-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
160 lines (130 loc) · 3.59 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
package main
import (
"context"
"flag"
"os"
"strings"
"sync"
"sync/atomic"
"time"
"cloud.google.com/go/datastore"
log "github.com/sirupsen/logrus"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"google.golang.org/grpc"
)
func newDatastoreClient(ctx context.Context) *datastore.Client {
projectID := os.Getenv("DATASTORE_PROJECT_ID")
if len(projectID) == 0 {
log.Fatal("DATASTORE_PROJECT_ID env variable is empty")
}
client, err := datastore.NewClient(
ctx,
projectID,
option.WithGRPCDialOption(grpc.WithBackoffMaxDelay(5*time.Second)),
option.WithGRPCDialOption(grpc.WithBlock()),
option.WithGRPCDialOption(grpc.WithTimeout(10*time.Second)),
)
if err != nil {
log.Fatalf("cannot create Datastore client for project %s: %v", projectID, err)
}
return client
}
func putDocument(client *datastore.Client, doc document) error {
var err error
for i := 0; i < 6; i++ {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
_, err := client.Put(ctx, doc.ID, &doc)
cancel()
if err == nil {
return nil
}
time.Sleep(time.Duration(i+3) * time.Second)
}
return err
}
func displayProgress(updated, total int) {
if total <= 0 {
log.Infof("progress: %d", updated)
} else {
log.Infof("progress: %d of %d (%d%%)", updated, total, (100 * int(updated) / total))
}
}
func main() {
// read args
var skip, emitProgressEvery uint64
var dsKind string
var allowAttributeDeletion, skipCount bool
var numWorkers int
flag.StringVar(&dsKind, "kind", "", "Datastore kind")
flag.BoolVar(&allowAttributeDeletion, "allowAttributeDeletion", false, "Allow writing a document with less fields than the original one")
flag.BoolVar(&skipCount, "skipCount", false, "Skip initial count")
flag.IntVar(&numWorkers, "workers", 4, "The number of workers to use")
flag.Uint64Var(&skip, "skip", 0, "skip the first N documents")
flag.Uint64Var(&emitProgressEvery, "emitProgressEvery", 1000, "Display progress every N records")
flag.Parse()
if dsKind == "" {
flag.Usage()
os.Exit(0)
}
// connect to google Datastore
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
client := newDatastoreClient(ctx)
defer client.Close()
// the main query
query := datastore.NewQuery(dsKind)
// count
var total int
var err error
if !skipCount {
log.Infof("main: count started, kind=%s", dsKind)
total, err = client.Count(context.Background(), query)
if err != nil {
log.Fatalf("main: count failed: %v", err)
}
log.Infof("main: count completed, total=%d", total)
}
// spawn write workers
var wg sync.WaitGroup
var updated uint64
chanSize := numWorkers * 50 // arbitrary value
if total <= chanSize {
chanSize = 0
}
docs := make(chan document, chanSize)
for i := 0; i < numWorkers; i++ {
wg.Add(1)
go func() {
for d := range docs {
currentIndex := atomic.AddUint64(&updated, 1)
if currentIndex >= skip {
if err := putDocument(client, d); err != nil {
log.Fatalf("main: putDocument failed, counter=%d, err=%v", updated, err)
}
}
if updated%emitProgressEvery == 0 {
displayProgress(int(updated), total)
}
}
wg.Done()
}()
}
// run the main query, iterate all the docs and push them in the docs channel
it := client.Run(context.Background(), query)
for {
var doc document
_, err := it.Next(&doc)
if err == iterator.Done {
break
}
noStructErr := err != nil && allowAttributeDeletion && strings.Contains(err.Error(), "no such struct field")
if err != nil && !noStructErr {
log.Fatal(err)
}
docs <- doc
}
close(docs)
wg.Wait()
log.Infof("main: done")
}