1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# A four-node zc mesh where each node belongs to a different person.
#
# ./mesh/provision.sh # mint 4 accounts + keys
# docker compose -f mesh/compose.yaml up -d --build
# ./mesh/verify.sh
#
# Nodes 1-3 are other people's machines. Node 4 is yours. The only thing that
# makes node4 "yours" is the key it carries -- the containers are identical --
# and that is the point: ownership here is an identity, not a kind of hardware.
#
# owner ip host port price/hr gpu
# node1 alice 10.42.0.11 19001 1.80 RTX 4090
# node2 bob 10.42.0.12 19002 3.60 A100 40GB
# node3 carol 10.42.0.13 19003 7.20 H100 80GB
# node4 YOU 10.42.0.14 19004 2.50 RTX 4090
#
# The prices differ deliberately. With one shared price the router has nothing
# to choose between and "which node won" carries no information; spread over 4x
# it does.
#
# Host ports are 19001-19004, clear of the marketplace stack's 8000 (api), 5173
# (web) and 9000/9001 (MinIO). Both stacks have to run at once for billing to
# work, so overlapping here would be a hard conflict rather than a nuisance.
x-node:
build:
# Repo root: the build needs Cargo.toml and src/, which are above mesh/.
context: ..
dockerfile: mesh/Dockerfile
# Reads GH_TOKEN from your shell. Needed only to FETCH sources: cargo
# resolves the private zakuro-drive git dep even though the `hooks` feature
# that uses it is off, so a build without it dies with git exit 128. See the
# long comment in mesh/Dockerfile.
# export GH_TOKEN=$(gh auth token)
secrets:
- gh_token
image: zakuroai/zc-mesh:dev
restart: unless-stopped
# The marketplace API runs in a different compose project, so it is reachable
# only via the host. host-gateway makes this name work on Linux too, where it
# is not built in as it is on Docker Desktop.
extra_hosts:
- "host.docker.internal:host-gateway"
x-node-env:
ZAKURO_P2P: "true"
# Shared secret for the private /peer/* API, and it must be byte-identical on
# all four. peer_handshake_auth 401s any non-loopback caller whose X-Peer-Key
# does not match, and a rejected probe is not an error -- the node simply
# learns no peers. A typo here yields four healthy, isolated brokers.
ZAKURO_PEER_KEY: "dev-mesh-peer-key"
# The billing authority. With api_url + api_key set, is_billing_enabled()
# turns on and reserve/commit go to the marketplace; the four keys below then
# name four real accounts whose balances actually move.
#
# Deliberately NOT ZAKURO_MASTER_KEY. The master key is the standalone
# alternative, and extract_user_from_key_format() maps it to "admin" -- so
# setting it would silently collapse alice, bob, carol and you into one
# account while everything still appeared to work. entrypoint.sh refuses to
# boot if it is set.
ZAKURO_API_URL: "http://host.docker.internal:8000"
# The hash ring is built once in PeerManager::new_with_node_key and never
# mutated, so ring membership must not depend on which peers happened to
# answer during startup. With discovery on, nodes booting at different moments
# disagree about total_brokers and therefore about who owns whose credits.
# An explicit peer list makes every ring identical and boot-order independent.
ZAKURO_DISCOVER_BROKER_PEERS: "false"
ZAKURO_SCAN_INTERVAL: "5"
ZAKURO_WAL_PATH: "/var/zakuro/wal.jsonl"
MW_PORT: "3960"
MW_BIND: "0.0.0.0" # peers dial the worker by node IP, not loopback
MW_MIN_CHARGE: "0.01"
services:
node1:
<<: *node
container_name: zk-mesh-node1
hostname: node1
environment:
<<: *node-env
ZAKURO_NODE_NAME: node1
ZAKURO_API_KEY: ${MESH_KEY_NODE1:?run ./mesh/provision.sh first}
ZAKURO_MESH_IP: 10.42.0.11
ZAKURO_PEERS: "10.42.0.12:9000,10.42.0.13:9000,10.42.0.14:9000"
MW_NAME: worker-alice
MW_PRICE_PER_HOUR: "1.80"
MW_CPUS: "8"
MW_MEMORY_GIB: "32"
MW_GPU_MODEL: "RTX 4090"
MW_GPU_VRAM_GIB: "24"
networks:
zk-mesh:
ipv4_address: 10.42.0.11
ports:
- "19001:9000"
volumes:
- node1-wal:/var/zakuro
node2:
<<: *node
container_name: zk-mesh-node2
hostname: node2
environment:
<<: *node-env
ZAKURO_NODE_NAME: node2
ZAKURO_API_KEY: ${MESH_KEY_NODE2:?run ./mesh/provision.sh first}
ZAKURO_MESH_IP: 10.42.0.12
ZAKURO_PEERS: "10.42.0.11:9000,10.42.0.13:9000,10.42.0.14:9000"
MW_NAME: worker-bob
MW_PRICE_PER_HOUR: "3.60"
MW_CPUS: "16"
MW_MEMORY_GIB: "64"
MW_GPU_MODEL: "A100 40GB"
MW_GPU_VRAM_GIB: "40"
networks:
zk-mesh:
ipv4_address: 10.42.0.12
ports:
- "19002:9000"
volumes:
- node2-wal:/var/zakuro
node3:
<<: *node
container_name: zk-mesh-node3
hostname: node3
environment:
<<: *node-env
ZAKURO_NODE_NAME: node3
ZAKURO_API_KEY: ${MESH_KEY_NODE3:?run ./mesh/provision.sh first}
ZAKURO_MESH_IP: 10.42.0.13
ZAKURO_PEERS: "10.42.0.11:9000,10.42.0.12:9000,10.42.0.14:9000"
MW_NAME: worker-carol
MW_PRICE_PER_HOUR: "7.20"
MW_CPUS: "32"
MW_MEMORY_GIB: "128"
MW_GPU_MODEL: "H100 80GB"
MW_GPU_VRAM_GIB: "80"
networks:
zk-mesh:
ipv4_address: 10.42.0.13
ports:
- "19003:9000"
volumes:
- node3-wal:/var/zakuro
# Yours. Send your jobs here: with remote_only the broker must place them on
# someone else's worker, so your balance falls and theirs rises. Sent to any
# of the other three, the same job would debit YOU and credit them just the
# same -- the requester is whoever the Bearer key names, not whoever owns the
# door you knocked on.
node4:
<<: *node
container_name: zk-mesh-node4
hostname: node4
environment:
<<: *node-env
ZAKURO_NODE_NAME: node4
ZAKURO_API_KEY: ${MESH_KEY_NODE4:?run ./mesh/provision.sh first}
ZAKURO_MESH_IP: 10.42.0.14
ZAKURO_PEERS: "10.42.0.11:9000,10.42.0.12:9000,10.42.0.13:9000"
MW_NAME: worker-mine
MW_PRICE_PER_HOUR: "2.50"
MW_CPUS: "8"
MW_MEMORY_GIB: "32"
MW_GPU_MODEL: "RTX 4090"
MW_GPU_VRAM_GIB: "24"
networks:
zk-mesh:
ipv4_address: 10.42.0.14
ports:
- "19004:9000"
volumes:
- node4-wal:/var/zakuro
secrets:
gh_token:
environment: GH_TOKEN
networks:
zk-mesh:
name: zk-mesh
ipam:
config:
# 10.42.0.0/24 and deliberately NOT 10.13.13.0/24. vpn::is_mesh_ip()
# matches 10.13.13.x exactly, and peer fetches for a matching host are
# routed through vpn::mesh_proxy_addr() -- a CONNECT sidecar that does
# not exist here. Staying off that range keeps the proxy path
# unreachable instead of intermittently attempted.
- subnet: 10.42.0.0/24
volumes:
node1-wal:
node2-wal:
node3-wal:
node4-wal: