sightingdb 0.7.2

# SightingDB configuration.
#
# Comments here are preserved: the management interface never rewrites this
# file. It writes only the separate files named below — acl_file, tiers_file,
# tags_file and [galaxy] peers_file — each of which it owns and rewrites whole.
# Anything set directly in this file is therefore read-only in the interface,
# which will say so rather than appearing to accept a change.

[daemon]
# Set to false to run without the HTTP API (DNS or ZMQ ingest only).
enabled = true
listen_ip = "0.0.0.0"
listen_port = 9999

# Require an API key on the sighting API. The management interface always
# requires one regardless of this setting.
authenticate = false

# Detaching is opt-in; under systemd or Docker leave this false and let the
# supervisor do it. See etc/sightingdb.service.
daemonize = false

ssl = true
ssl_cert = "ssl/cert.pem"
ssl_key = "ssl/key.pem"

# Largest accepted bulk POST body, in bytes.
post_limit = 2_500_000_000

# Only used when daemonize = true.
log_out = "/var/log/sightingdb/sightingdb.log"
log_err = "/var/log/sightingdb/sightingdb.error.log"

# Where the database is snapshotted. Comment this out to run purely in memory,
# losing everything when the process stops.
dbdir = "/var/lib/sighting"

# Seconds between snapshots. The database is always saved on a clean shutdown;
# this bounds how much a crash can lose. 0 saves only on shutdown.
#
# Storage is split into one zstd-compressed file per top-level namespace, so
# "myorg/thisone" lives in myorg-<hash>.json.zst and a periodic snapshot only
# rewrites the files that changed. Everything internal (_all, _shadow, _config)
# shares sightingdb.json.zst.
snapshot_interval = 300

# zstd level for those files. 3 is the knee of the curve: measured on a 50 MB
# snapshot it reaches 7.3x in 0.07s, where level 19 takes 25s for 11.3x. Higher
# levels cost compression time on every save for very little space.
compression_level = 3

# Seconds between eviction sweeps, which reclaim expired values and the
# namespaces left empty by them. 0 disables the sweeper.
sweep_interval = 60

# Hourly statistics buckets kept per value. 720 is 30 days. Without a limit
# every value gains a bucket per hour for as long as it is written to.
# 0 keeps all of them.
stats_retention = 720

# How long a shadow sighting (the record that something was searched for) is
# kept, in seconds. 0 keeps them forever, so `_shadow/*` grows without bound.
# 2592000 is 30 days.
shadow_ttl = 2_592_000

# How many rejected values to keep in memory, so that
# /_management/api/rejections can say which values were turned away and why.
# Bounded on purpose: a misbehaving feed can reject as fast as it can send.
# The oldest go first once the cap is reached, nothing survives a restart, and
# 0 switches the record off.
rejection_log = 1000

# This server's name in its own counters. Sightings are counted per node, so
# that merging two copies of a value is a sum of separate contributions rather
# than a replay of increments — an increment is not idempotent, and a sync that
# replayed one would double the count every time it ran.
#
# "local" is right until this server joins a galaxy. Then give each one its own
# name: two sharing an id would each take the other's contribution for their
# own, and a merge would lose one of them. Letters, digits, '-', '_' and '.',
# up to 64 characters.
#
# Changing it later is safe but not free: the old name keeps its contribution
# in the stored data and the new one starts at zero, so the total is unchanged
# while the attribution is split.
#node_id = "local"

# Which namespaces this server stores. Absent means all of them, which is what
# a standalone server wants and what every release before galaxies did.
#
#   namespaces = ["/"]                  everything, said explicitly
#   namespaces = ["feeds", "threats"]   those subtrees and nothing else
#   namespaces = []                     nothing of its own: a pure router
#
# Prefixes match whole path segments, so "feeds" holds feeds/misp/ips and never
# feeds-internal. The internal namespaces (_all, _shadow/*, _config) are always
# held whatever this says — a server cannot keep a consensus tally otherwise —
# so they must not be listed here.
#
# A write to a namespace this server does not store is refused with 421. Until
# forwarding exists, narrowing this is how you declare what a server is for,
# not a way to make it pass work along.
#namespaces = ["/"]

# How long each top-level namespace stays in memory.
#
#   hot   never evicted
#   warm  written out and dropped once untouched for warm_idle seconds; any
#         read or write starts the window again
#   cold  dropped at the next sweep once idle
#
# "cold" does not mean a disk round trip per operation: a cold shard is loaded
# on demand and dropped at the next sweep, so a burst of writes costs one load
# rather than one per write. Internal state (_all, _config) is always hot,
# whatever is configured, because consensus and API keys are consulted
# constantly.
#
# API keys live in their own file because the management interface rewrites it
# whenever a key is created or revoked. Without this, keys can be declared in an
# [acl] table below, but the interface will not be able to edit them.
acl_file = "acl.toml"

# How tags are shown in the management interface: a colour and a description
# each. Its own file for the same reason as the keys — the interface rewrites it
# whenever a tag is edited.
#
# This decides presentation only. A value can carry a tag nothing here defines,
# and that is not an error: a feed brings whatever tags it brings, and refusing
# them to keep a table tidy would lose data. Undefined tags are listed in the
# interface alongside the defined ones so they can be adopted.
#
# A name ending in ":" is a family and colours everything beneath it, so
# "stix-type:" covers stix-type:ipv4-addr and every other value of that key.
#
# A new installation is seeded with the five TLP 2.0 labels — clear, green,
# amber, amber+strict, red — plus tlp:white, which 2.0 renamed to CLEAR,
# because sources still send it: CIRCL's OSINT feed tags events with both.
# WHITE is given CLEAR's colour, because that is what it means. A colour per
# family of SightingDB's own vocabulary comes with them. After that this file
# is yours: any of them can be recoloured or removed, and nothing puts them
# back.
#
# Without this setting those colours are still used, but are read-only in the
# interface.
tags_file = "tags.toml"

# Eviction needs somewhere to write, so it only happens when dbdir is set.
[storage]
default_tier = "hot"
warm_idle = 3600

# Rewritten by the management interface whenever a tier is changed there, which
# is why it is a file of its own: rewriting this one would discard its comments.
# Without it, tiers come from [storage.tiers] below and the interface can show
# them but not change them.
tiers_file = "tiers.toml"

# Overrides, keyed by the top-level namespace and covering everything under it.
# An entry is a tier, or a table setting the tier, the idle window, or both —
# so two warm shards can wait different lengths of time before being written
# out. Changed from the management interface or POST /_api/tier as well, which
# rewrites tiers_file.
# [storage.tiers]
# myorg = "hot"
# archive = "cold"
# occasional = { tier = "warm", warm_idle = 86400 }
# staging = { warm_idle = 300 }

# API keys, used only when acl_file is not set.
#
#   "<apikey>" = "<grant>[, <grant>...]"
#
# A grant is r, w or rw, optionally scoped with :<namespace prefix>, plus
# "admin" for access to the management interface at /_management/. Prefixes
# match whole path segments, so rw:feeds/misp covers feeds/misp/ips but not
# feeds/misp-internal.
#
# [acl]
# changeme = "rw, admin"
# analyst = "r"
# feed-misp = "rw:feeds/misp"

# ---------------------------------------------------------------------------
# Answer sightings over DNS, using the DNSBL conventions security tooling
# already speaks.
#
# READ THIS BEFORE ENABLING: DNS has no authentication. The ACL does not apply,
# so anything reachable here is readable by anyone who can send a UDP packet.
# Only the namespaces listed below answer, the listener binds to loopback unless
# you change it, and rate_limit exists because an open UDP responder is a
# reflection amplifier.
# ---------------------------------------------------------------------------
# [dns]
# enabled = true
# listen_ip = "127.0.0.1"
# listen_port = 5353
# zone = "sdb.example.com"
# TTL on answers, and how long resolvers cache a "never seen" reply.
# ttl = 60
# Queries per second per source address; 0 disables the limit.
# rate_limit = 100
# threads = 2
# Whether a DNS lookup counts as a search. Off by default: it is an
# unauthenticated write path into _shadow/*.
# shadow = false
#
# How a value is spelled in the query name:
#   ip      4.3.2.1.malware.sdb.example.com  -> 1.2.3.4   (DNSBL convention)
#   domain  evil.com.domains.sdb.example.com -> evil.com
#   base32  <base32 of the value>.hashes.sdb.example.com
# [dns.namespaces]
# malware = { namespace = "malware/ips", encoding = "ip" }
# domains = { namespace = "malware/domains", encoding = "domain" }
# hashes = { namespace = "malware/hashes", encoding = "base32" }

# ---------------------------------------------------------------------------
# Ingest sightings from a ZeroMQ publisher such as MISP's. This is a SUB socket
# that connects out to a publisher you trust; nothing is exposed.
# ---------------------------------------------------------------------------
# [zmq]
# enabled = true
# endpoint = "tcp://misp.example.com:50000"
# MISP publishes "<topic> <json>"; subscribing to a prefix matches it.
# topics = ["misp_json_attribute", "misp_json"]
# "misp" reads MISP attributes and events; "native" reads
# {"items":[{"namespace":..,"value":..,"timestamp":..}]}
# format = "misp"
# Only ingest attributes MISP flagged as actionable.
# require_to_ids = true
# Where unmapped attribute types go; omit to discard them.
# default_namespace = ""
# TTL on ingested sightings, and seconds between reconnection attempts.
# ttl = 0
# reconnect = 5
#
# [zmq.types]
# ip-src = "misp/ips"
# ip-dst = "misp/ips"
# domain = "misp/domains"
# hostname = "misp/domains"
# url = "misp/urls"
# md5 = "misp/hashes"
# sha1 = "misp/hashes"
# sha256 = "misp/hashes"

# ---------------------------------------------------------------------------
# STIX 2.1, both ways.
#
# Importing with --import-stix <file or directory>: counts and time windows come
# from the bundle, so an observed-data seen 12 times between two instants
# becomes 12 sightings spanning that window. What the bundle says about a value
# — its type, markings, confidence, who saw it — is kept as tags.
#
# Exporting with GET /stix/<namespace>, or the management interface's Export
# STIX button: identity and identity_class say who the bundle is published as.
# The default is "SightingDB" as a "system", which is a database reporting what
# it saw rather than an organisation vouching for it.
#
# The types table below is read both ways: a namespace mapped for import is a
# namespace whose values the export knows the type of, whichever way they
# arrived.
# ---------------------------------------------------------------------------
# [stix]
# default_namespace = ""
# ttl = 0
# identity = "Alpha Threat Analysis Org."
# identity_class = "organization"
#
# Keys containing a dot must be QUOTED, or TOML reads them as a nested table.
# [stix.types]
# ipv4-addr = "stix/ips"
# ipv6-addr = "stix/ips"
# domain-name = "stix/domains"
# url = "stix/urls"
# "file.MD5" = "stix/hashes"
# "file.SHA-1" = "stix/hashes"
# "file.SHA-256" = "stix/hashes"


# ---------------------------------------------------------------------------
# [galaxy] — the other servers this one knows about
# ---------------------------------------------------------------------------
#
# Each peer carries the key this server authenticates to it with, and that key
# is the bound on what this server may do there: a key granted "rw:feeds" on
# the peer cannot write anywhere else, however this server is configured or
# compromised. Give each one the narrowest grant that does the job.
#
# A peer may itself have peers, so a router in front of routers is a cascade.
# max_hops is what stops a miswired cycle: a cycle inflates every count that
# travels round it, so this is a limit rather than a tuning knob.
#
# A server that does not store a namespace forwards the request to the peers
# that do: writes to every live mirror, reads to one chosen by hashing the
# value so the same value always comes from the same place.
#
# `namespaces` on a peer says what it holds; absent means a full mirror. It is
# declared here rather than asked of the peer, because asking would need the
# peer key to carry an admin grant and the point of that key is to be the
# narrowest thing that works. It must agree with the peer's own
# [storage] namespaces.
#
#[galaxy]
#max_hops = 4
#health_interval = 30
# Seconds between catch-up passes: a server that was down finds what it
# missed and pulls it from a peer that was up. 0 switches it off, which
# leaves such a server behind until someone syncs it by hand.
#sync_interval = 300
# Seconds between rebuilds of the consensus tally from the galaxy. A server
# in front of one counts a value towards consensus when a mirror says the
# sighting was new there, which only ever rises — values expire and
# namespaces are deleted on the nodes, and neither reaches it. This surveys
# the galaxy and puts the tally right, so it is a repair rather than a
# steady cost. 0 switches it off.
#reconcile_interval = 3600
# Seconds between offering this server's keys to its peers, so a peer that
# was down for a key change picks it up. A change is pushed as it happens
# regardless; this only catches up what was missed. 0 switches the periodic
# offer off.
#
# A push goes through the peer's own management interface, so the peer's ACL
# decides: this server can only administer a peer it holds an admin key for.
# Note that the offer is additive — a revocation made while a peer was down
# does not reach it, and must be repeated.
#gossip_interval = 600
#
# One place to manage keys. The offer above is additive and so cannot carry a
# revocation to a server that was offline for it. Set acl_authority on the
# server that owns the galaxy's keys and acl_replaceable on each server that
# accepts them, and the owner's offer becomes the whole list, to be held
# exactly — which does remove.
#
# Both ends must agree, so a server with acl_replaceable off keeps its own
# keys and nothing is lost by not opting in. Setting both on one server is
# refused at startup: they would take turns overwriting each other.
#acl_authority = false
#acl_replaceable = false
#verify_tls = true
#
# Peers added from the management interface are written here, and are in effect
# as soon as they are saved — adding a mirror should not need a restart, since
# restarting a router is a gap in service for everything behind it.
#
# A file of its own, again because this one is yours: peers written into the
# `peers` list below are shown in the interface but not editable there, as a
# change made there would only last until the next restart. Set this and the
# interface can manage its own peers; leave it out and the galaxy is read-only
# there.
#peers_file = "peers.toml"
#peers = [
#  { url = "https://node-a.example:9999", key = "..." },
#  { url = "https://node-b.example:9999", key = "...", namespaces = ["feeds"] },
#  # Kept, and sent nothing: no requests, no catch-up, not even a health
#  # probe. For a node out of service that you do not want to forget how to
#  # reach. A namespace only it held is out of reach while it is off.
#  { url = "https://spare.example:9999", key = "...", enabled = false },
#]