opsqueue 0.30.1

lightweight batch processing queue for heavy loads
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

SQLite format 3@  �.zp��
K�	�	`b�W�`�LD�?1�indexsubmissions_failed_prefixsubmissions_failedCREATE INDEX submissions_failed_prefix ON submissions_failed (prefix, id)�
E7�+indexsubmissions_completed_prefixsubmissions_completedCREATE INDEX submissions_completed_prefix ON submissions_completed (prefix, id)e
1#�indexsubmissions_prefixsubmissionsCREATE INDEX submissions_prefix ON submissions (prefix, id)�
3�[indexrandom_chunks_orderchunks
CREATE INDEX random_chunks_order ON chunks (
      random_order
    , submission_id
    , chunk_index
)�f''�tablechunks_failedchunks_failedCREATE TABLE chunks_failed
(
    submission_id INTEGER NOT NULL,
    chunk_index INTEGER NOT NULL,
    input_content BLOB NULL,
    failure BLOB NOT NULL DEFAULT '',
    failed_at DATETIME NOT NULL, -- Unix Timestamp
    skipped BOOL NOT NULL DEFAULT false,

    PRIMARY KEY (submission_id, chunk_index)
) WITHOUT ROWID�
--�qtablechunks_completedchunks_completedCREATE TABLE chunks_completed
(
    submission_id INTEGER NOT NULL,
    chunk_index INTEGER NOT NULL,
    output_content BLOB NULL, -- Unset iff only (submission_id, id) + submission's metadata is enough to figure out content of Chunk from object_storage
    completed_at DATETIME NOT NULL, -- Unix Timestamp

    PRIMARY KEY (submission_id, chunk_index)
) WITHOUT ROWID�{	�Qtablechunkschunks
CREATE TABLE chunks
(
    submission_id INTEGER NOT NULL,
    chunk_index INTEGER NOT NULL,
    input_content BLOB NULL, -- Unset iff only (submission_id, id) + submission's metadata is enough to figure out content of Chunk from object_storage
    retries INTEGER NOT NULL DEFAULT 0, random_order INTEGER NOT NULL GENERATED ALWAYS AS (
    (((submission_id + chunk_index) % 65536) * 40503) % 65536
    ) VIRTUAL,

    PRIMARY KEY (submission_id, chunk_index)
) WITHOUT ROWID, STRICTCW1indexsqlite_autoindex_submissions_failed_1submissions_failed	�11�itablesubmissions_failedsubmissions_failedCREATE TABLE submissions_failed
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    metadata BLOB,
    failed_chunk_id INTEGER NOT NULL, -- NOTE: There might in rare cases be multiple concurrent failures, but only one of them will be 'first' (ordered by who gets write access to the submissions table lock first), which is the one stored here.
    failed_at DATETIME NOT NULL -- Unix Timestamp
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}")I]7indexsqlite_autoindex_submissions_completed_1submissions_completed�677�tablesubmissions_completedsubmissions_completedCREATE TABLE submissions_completed
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    metadata BLOB,
    completed_at DATETIME NOT NULL -- Unix Timestamp
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}")5I#indexsqlite_autoindex_submissions_1submissions�##�ctablesubmissionssubmissionsCREATE TABLE submissions
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    chunks_done INTEGER NOT NULL DEFAULT 0,
    metadata BLOB
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}")?S-indexsqlite_autoindex__sqlx_migrations_1_sqlx_migrations�)--�table_sqlx_migrations_sqlx_migrationsCREATE TABLE _sqlx_migrations (
    version BIGINT PRIMARY KEY,
    description TEXT NOT NULL,
    installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    success BOOLEAN NOT NULL,
    checksum BLOB NOT NULL,
    execution_time BIGINT NOT N	

*�E�n
�
�
*c-3	lj��j-track chunk size2025-04-16 08:45:57�������-��ԧ�x���>$�4��e���,EA�����M��c%nq�i93	lj�Cͦadd strategic metadata2025-01-10 10:25:17Z+5!]�@^��ׄ�P�T�����j8��}�TR�V�� F@��rO�rK3	lh���add tracecontext to submissions2025-01-04 16:07:57<�{�?��Ð���hE�}}P<��vƩ��U��.�¯9�ʈ�3�s1L+�|mA3	lh��Jdsubmission index by prefix2025-01-04 16:07:57(i�'p�y/ɇ���v�ƻ���ڮ��r,�a��_����7���R�5/\Kf33	lh����random chunks index2025-01-04 16:07:57#L�7.��Xn�\��Z�%�uYY����|�e���}��,���>A<��Y�9�Y3	lh��Xachunks2025-01-04 16:07:57�Q���R��DO59�U0S��+�x�5]������"Hq�h+�!LwH�2d�$&�^#3	lh��W�submissions2025-01-04 16:07:57�*��ƨg/��0�hdI��`f���0��V����w1��<�!��HA��3$�
��������
j��j-
j�Cͦ
h���
h��Jd
h����
h��Xa		h��W�














	�	���=
K���	`b�W�`�LD�?1�indexsubmissions_failed_prefixsubmissions_failedCREATE INDEX submissions_failed_prefix ON submissio�311�tablesubmissions_failedsubmissions_failedCREATE TABLE submissions_failed
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    metadata BLOB,
    failed_chunk_id INTEGER NOT NULL, -- NOTE: There might in rare cases be multiple concurrent failures, but only one of them will be 'first' (ordered by who gets write access to the submissions table lock first), which is the one stored here.
    failed_at DATETIME NOT NULL -- Unix Timestamp
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}", chunk_size INTEGER)�J77�3tablesubmissions_completedsubmissions_completedCREATE TABLE submissions_completed
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    metadata BLOB,
    completed_at DATETIME NOT NULL -- Unix Timestamp
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}", chunk_size INTEGER)�"##�tablesubmissionssubmissionsCREATE TABLE submissions
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    chunks_done INTEGER NOT NULL DEFAULT 0,
    metadata BLOB
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}", chunk_size INTEGER)�{	�Qtablechunkschunks
CREATE TABLE chunks
(
    submission_id INTEGER NOT NULL,
    chunk_index INTEGER NOT NULL,
    input_content BLOB NULL, -- Unset iff only (submission_id, id) + submission's metadata is enough to figure out content of Chunk from object_storage
    retries INTEGER NOT NULL DEFAULT 0, random_order INTEGER NOT NULL GENERATED ALWAYS AS (
    (((submission_id + chunk_index) % 65536) * 40503) % 65536
    ) VIRTUAL,

    PRIMARY KEY (submission_id, chunk_index)
) WITHOUT ROWID, STRICTCW1indexsqlite_autoindex_submissions_failed_1submissions_failed	"11�itablesubmissions_failedsubmissions_failedCREATE TABLE submissions_failed
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    metadata BLOB,
    failed_chunk_id INTEGER NOT NULL, -- NOTE: There might in rare cases be multiple concurrent failures, but only one of them will be 'first' (ordered by who gets write access to the submissions table lock first), which is the one stored here.
    failed_at DATETIME NOT NULL -- Unix Timestamp
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}")I]7indexsqlite_autoindex_submissions_completed_1submissions_completed
�977�tablesubmissions_completedsubmissions_completedCREATE TABLE submissions_completed
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    metadata BLOB,
    completed_at DATETIME NOT NULL -- Unix Timestamp
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}")5I#indexsqlite_autoindex_submissions_1submissions##�ctablesubmissionssubmissionsCREATE TABLE submissions
(
    id BIGINT PRIMARY KEY NOT NULL,
    prefix TEXT,
    chunks_total INTEGER NOT NULL DEFAULT 0,
    chunks_done INTEGER NOT NULL DEFAULT 0,
    metadata BLOB
, otel_trace_carrier TEXT NOT NULL DEFAULT "{}")?S-indexsqlite_autoindex__sqlx_migrations_1_sqlx_migrations�)--�table_sqlx_migrations_sqlx_migrationsCREATE TABLE _sqlx_migrations (
    version BIGINT PRIMARY KEY,
    description TEXT NOT NULL,
    installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    success BOOLEAN NOT NULL,
    checksum BLOB NOT NULL,
    execution_time BIGINT NOT NULL
)
�^�e�n
�v�u���[G+�Qindexrandom_chunks_metadata_order2chunks_metadataCREATE INDEX random_chunks_metadata_order2 ON chunks_metadata (
      metadata_key
    , random_order
    , metadata_value
    , submission_id
    , chunk_index
)�YE+�Oindexrandom_chunks_metadata_orderchunks_metadataCREATE INDEX random_chunks_metadata_order ON chunks_metadata (
      metadata_key
    , metadata_value
    , random_order
    , submission_id
    , chunk_index
)�;++�-tablechunks_metadatachunks_metadataCREATE TABLE chunks_metadata
(
    submission_id INTEGER NOT NULL,
    chunk_index INTEGER NOT NULL,
    metadata_key TEXT NOT NULL,

    metadata_value INTEGER NOT NULL, random_order INTEGER NOT NULL GENERATED ALWAYS AS (
    (((submission_id + chunk_index) % 65536) * 40503) % 65536
) VIRTUAL,

    -- auto-delete when chunk completes or fails:
    FOREIGN KEY (submission_id, chunk_index) REFERENCES chunks ON DELETE CASCADE,

    PRIMARY KEY (submission_id, chunk_index, metadata_key, metadata_value)
) WITHOUT ROWID, STRICT�@G5�indexlookup_submission_by_metadatasubmissions_metadataCREATE INDEX lookup_submission_by_metadata ON submissions_metadata (
      metadata_key
    , metadata_value
    , submission_id
)�q55�tablesubmissions_metadatasubmissions_metadataCREATE TABLE submissions_metadata
(
    submission_id INTEGER NOT NULL,
    metadata_key TEXT NOT NULL,

    -- We only support 64-bit integers as values,
    -- for efficiency and reduced storage.
    --
    -- Since we do not depend on this value for ordering (Ord) but only for filtering (Eq),
    -- it's okay to transform a u64 into a i64 before saving,
    -- so we can truly accept the full 64 bit range here.
    metadata_value INTEGER NOT NULL,

    -- NOTE: We don't set a foreign key here
    -- because the submission_id might point to
    -- a record in one of multiple tables
    -- (and we want to keep the metadata available also for completed/failed submissions)

    -- Quickly select all metadata of a particular submission
    PRIMARY KEY (submission_id, metadata_key, metadata_value)
) WITHOUT ROWID, STRICT�?1�indexsubmissions_failed_prefixsubmissions_failedCREATE INDEX submissions_failed_prefix ON submissions_failed (prefix, id)�
E7�+indexsubmissions_completed_prefixsubmissions_completedCREATE INDEX submissions_completed_prefix ON submissions_completed (prefix, id)e
1#�indexsubmissions_prefixsubmissionsCREATE INDEX submissions_prefix ON submissions (prefix, id)�
3�[indexrandom_chunks_orderchunks
CREATE INDEX random_chunks_order ON chunks (
      random_order
    , submission_id
    , chunk_index
)�f''�tablechunks_failedchunks_failedCREATE TABLE chunks_failed
(
    submission_id INTEGER NOT NULL,
    chunk_index INTEGER NOT NULL,
    input_content BLOB NULL,
    failure BLOB NOT NULL DEFAULT '',
    failed_at DATETIME NOT NULL, -- Unix Timestamp
    skipped BOOL NOT NULL DEFAULT false,

    PRIMARY KEY (submission_id, chunk_index)
) WITHOUT ROWID�
--�qtablechunks_completedchunks_completedCREATE TABLE chunks_completed
(
    submission_id INTEGER NOT NULL,
    chunk_index INTEGER NOT NULL,
    output_content BLOB NULL, -- Unset iff only (submission_id, id) + submission's metadata is enough to figure out content of Chunk from object_storage
    completed_at DATETIME NOT NULL, -- Unix Timestamp

    PRIMARY KEY (submission_id, chunk_index)
) WITHOUT ROWID