-
Notifications
You must be signed in to change notification settings - Fork 0
/
platform.proto
130 lines (108 loc) · 4.31 KB
/
platform.proto
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
syntax = "proto3";
package rkcy;
option go_package = "github.com/lachlanorr/rocketcycle/pkg/rkcypb";
import "google/protobuf/timestamp.proto";
import "apecs.proto";
message PlatformDef {
string name = 1;
string environment = 2;
repeated Concern concerns = 3;
repeated Cluster clusters = 4;
repeated StorageTarget storage_targets = 5;
google.protobuf.Timestamp update_time = 6;
}
message Concern {
Type type = 1;
string name = 2;
rkcy.Severity log_level = 3;
repeated Topics topics = 4;
// Type implies what topics will be present
enum Type {
// Required topics:
// * admin - handles control messages to coordinate distributed tasks
// * error - errors encountered during processing written here
// * ... - additional concern specific topics
GENERAL = 0;
// Required topics:
// * admin - handles control messages to coordinate distributed tasks
// * error - errors encountered during processing written here
// * ... - additional concern specific topics for stages of batch processing
// For telemetry purposes, stage topics are assumed to be in alphabetical order.
// Batch jobs topics aren't created by default, but on demand with the timestamp
// included in the names.
BATCH = 1;
// Required topics:
// * admin - handles control messages to coordinate distributed tasks
// * process - handles messages that affect internal state of models
// * error - errors encountered during processing written here
// * complete - completed transactions written here for post processing
// * storage - handles messages sent to the persistence layer
APECS = 2;
}
message Topics {
// base name, it will get decorated with additional dot notated pieces
string name = 1;
// current vs future transition state
State state = 2;
// current physical topic
Topic current = 3;
// topic we're in the process of migrating to, will be
// null unless an active migration is taking place
Topic future = 4;
repeated Program consumer_programs = 5;
// States surrounding current vs future topics and current to future transition
enum State {
// no future topic exists, everyting using current
CURRENT = 0;
// future topic added to concern and gets created
FUTURE_INIT = 1;
// producers all pause
PRODUCER_PAUSE = 2;
// consumers read until end and shutdown
CONSUMER_SHUTDOWN = 3;
// new consumers start on future, advanced passed newest
CONSUMER_FUTURE_START = 4;
// producers unpause and produce to future
PRODUCER_FUTURE_START = 5;
// future becomes current, future is nulled, state set to CURRENT
FUTURE_TO_CURRENT = 6;
}
}
message Topic {
// incrementing generation id, +1 every time we migrate a logical topic
int32 generation = 1;
// kafka cluster topic exists within
string cluster = 2;
int32 partition_count = 3;
// How to determine which partiton messages are produced to
PartitionStrategy partition_strat = 4;
}
// Different types of partitioning mechanisms
enum PartitionStrategy {
// FNV-1 64 bit has followed by modulus of partition count
FNV64_MOD = 0;
}
}
message Cluster {
// unique name of cluster
string name = 1;
// kafka brokers for the cluster
string brokers = 2;
// brokers used for admin topics, only one cluster can have this set
bool is_admin = 3;
}
message StorageTarget {
string name = 1;
string type = 2;
bool is_primary = 3;
map<string,string> config = 4;
}
message Program {
string name = 1;
repeated string args = 2;
string abbrev = 3;
map<string, string> tags = 4;
}