#if !defined(__WIN32__) && (defined(WIN32) || defined(_WIN32))
# define __WIN32__
#endif
#include "sqlite3.h"
#include "bcvencrypt.h"
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned short u16;
#ifdef __WIN32__
# include <winsock2.h>
# include <windows.h>
# include <ws2tcpip.h>
# define BCV_SOCKET_TYPE SOCKET
#else
# include <unistd.h>
# include <sys/types.h>
# include <sys/socket.h>
# include <arpa/inet.h>
# define BCV_SOCKET_TYPE int
# define O_BINARY 0
# define INVALID_SOCKET -1
#endif
#ifndef MSG_NOSIGNAL
# define MSG_NOSIGNAL 0
#endif
#include "bcvutil.h"
#include "bcvmodule.h"
#include "simplexml.h"
#include <curl/curl.h>
#define BCV_MANIFEST_FILE "manifest.bcv"
#define BCV_KV_FILE "bcv_kv.bcv"
#define BCV_DBID_SIZE 16
#define BCV_DBNAME_SIZE 128
#define BCV_MANIFEST_HEADER_BYTES 20
#define BCV_MANIFEST_DBHEADER_BYTES ( \
3*4+BCV_DBID_SIZE+BCV_DBNAME_SIZE \
)
#define BCV_MANIFEST_VERSION 4
#define BCV_MIN_NAMEBYTES 12
#define BCV_MAX_NAMEBYTES 32
#define BCV_MIN_MD5_NAMEBYTES 24
#define BCV_FSNAMEBYTES(namebytes) (((namebytes)*2)+4+1)
#define BCV_MAX_FSNAMEBYTES BCV_FSNAMEBYTES(BCV_MAX_NAMEBYTES)
#define NAMEBYTES(pManifest) ((pManifest)->nNamebytes)
#define GCENTRYBYTES(pManifest) (NAMEBYTES(pManifest) + 8)
#define BCV_CACHEFILE_NAME "cachefile.bcv\0"
#define BCV_DATABASE_NAME "blocksdb.bcv"
#define BCV_DEFAULT_NAMEBYTES 16
#define BCV_DEFAULT_BLOCKSIZE (4*1024*1024)
#define BCV_DEFAULT_HTTPTIMEOUT 600
#define BCV_LOCAL_KEYSIZE 16
#define HTTP_AUTH_ERROR 403
#define HTTP_PRECONDITION_FAILED 304
#define HTTP_NOT_FOUND 404
#define HTTP_BUSY_SNAPSHOT 412
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
typedef unsigned int u32;
typedef unsigned char u8;
typedef sqlite3_uint64 u64;
typedef sqlite3_int64 i64;
typedef struct ManifestHash ManifestHash;
typedef struct Manifest Manifest;
typedef struct ManifestDb ManifestDb;
typedef struct BcvCommon BcvCommon;
typedef struct Container Container;
typedef struct CacheEntry CacheEntry;
typedef struct BcvDispatch BcvDispatch;
typedef struct BcvContainer BcvContainer;
typedef struct BcvWrapper BcvWrapper;
typedef struct BcvBuffer BcvBuffer;
typedef struct BcvEncryptionKey BcvEncryptionKey;
typedef struct BcvIntKey BcvIntKey;
typedef struct BcvLog BcvLog;
struct Manifest {
int nRef;
int nDb;
int szBlk;
int nNamebytes;
u32 iMaxDbId;
ManifestDb *aDb;
int bDelFree;
int nDelBlk;
u8 *aDelBlk;
char *zETag;
u8 *pFree;
};
struct ManifestDb {
i64 iDbId;
i64 iParentId;
char zDName[BCV_DBNAME_SIZE];
int iVersion;
u8 *aBlkOrig;
int nBlkOrig;
int nBlkOrigAlloc;
u8 *aBlkLocal;
int nBlkLocal;
int nBlkLocalAlloc;
};
#define BCVFS_INSERT_BLOCK "REPLACE INTO block VALUES(?,?,?,?,?,?)"
#define BCVFS_INSERT_CONT "REPLACE INTO container VALUES(?,?,?,?,?,?)"
struct BcvCommon {
int bDaemon;
sqlite3_vfs *pVfs;
char *zDir;
i64 szBlk;
i64 nMaxCache;
Container *pCList;
CacheEntry **aHash;
CacheEntry *pUnused;
CacheEntry *pLruFirst;
CacheEntry *pLruLast;
i64 iLruTick;
int nHash;
int nBlk;
BcvLog *pLog;
sqlite3 *bdb;
sqlite3_stmt *pInsertBlock;
sqlite3_stmt *pInsertCont;
};
struct Container {
const char *zName;
const char *zStorage;
const char *zAccount;
const char *zContainer;
const char *zLocalDir;
Manifest *pMan;
int nClient;
Container *pNext;
int nBcv;
BcvWrapper *aBcv;
int eState;
u8 aKey[BCV_LOCAL_KEYSIZE];
BcvIntKey *pKey;
int iEnc;
};
struct BcvWrapper {
BcvContainer *pBcv;
BcvDispatch *pDisp;
int bUsable;
};
#define CONTAINER_STATE_NONE 0
#define CONTAINER_STATE_UPLOAD 1
#define CONTAINER_STATE_COPY 2
#define CONTAINER_STATE_POLL 3
#define CONTAINER_STATE_DELETE 4
struct CacheEntry {
int iPos;
int nRef;
int iEnc;
i64 iLruTick;
CacheEntry *pHashNext;
CacheEntry *pLruPrev;
CacheEntry *pLruNext;
u8 bValid;
u8 bDirty;
u8 nName;
u8 aName[BCV_MAX_NAMEBYTES];
};
struct BcvBuffer {
u8 *aData;
int nData;
int nAlloc;
};
struct ManifestHash {
int nHash;
u8 **aHash;
};
int bcvMHashBuild(Manifest*, int, ManifestDb*, ManifestHash**);
u8 *bcvMHashQuery(ManifestHash *pHash, u8 *aBlk, int nBlk);
void bcvMHashFree(ManifestHash *pHash);
void bcvBufferAppendU32(int *pRc, BcvBuffer *p, u32 iVal);
void bcvBufferAppendU64(int *pRc, BcvBuffer *p, u64 iVal);
void bcvBufferMsgString(int *pRc, BcvBuffer *p, const char *zStr);
void bcvBufferMsgBlob(int *pRc, BcvBuffer *p, const u8 *aData, int nData);
void bcvBufferZero(BcvBuffer *p);
void bcvBufferAppendRc(int *pRc, BcvBuffer *p, const void *a, int n);
int bcvStrcmp(const char *zLeft, const char *zRight);
int bcvStrlen(const char *z);
char *bcvMprintf(const char *zFmt, ...);
char *bcvMprintfRc(int *pRc, const char *zFmt, ...);
void *bcvMalloc(i64 nByte);
void *bcvRealloc(void *a, i64 nByte);
void *bcvMallocZero(i64 nByte);
u8 *bcvMemdup(int nIn, const u8 *aIn);
char *bcvStrdup(const char *zIn);
void *bcvMallocRc(int *pRc, i64 nByte);
char *bcvStrdupRc(int *pRc, const char *z);
i64 sqlite_timestamp(void);
int bcv_isdigit(char c);
int bcv_isspace(char c);
int bcvParseInt(const u8 *z, int n);
u32 bcvGetU16(const u8 *a);
void bcvPutU32(u8 *a, u32 iVal);
u32 bcvGetU32(const u8 *a);
void bcvPutU64(u8 *a, u64 iVal);
u64 bcvGetU64(const u8 *a);
int bcvManifestParse(u8 *a, int n, char *zETag, Manifest **ppOut, char **pz);
int bcvManifestParseCopy(
const u8 *a, int n,
const char *zETag,
Manifest **ppOut,
char **pz
);
i64 bcvManifestSize(Manifest*);
u8 *bcvManifestCompose(Manifest *p, int *pnOut);
void bcvManifestFree(Manifest *p);
void bcvManifestDeref(Manifest *p);
Manifest *bcvManifestRef(Manifest *p);
int bcvManifestDup(Manifest *p, Manifest **ppNew);
void bcvManifestExpand(int*, Manifest**, int);
void bcvBlockidToText(int nName, const u8 *pBlk, char *aBuf);
int bcvWritefile(sqlite3_file *pFd, const u8 *aData, int nData, i64 iOff);
int bcvOpenLocal(const char *, int bWal, int bReadonly, sqlite3_file **ppFd);
void bcvCloseLocal(sqlite3_file *pFd);
int bcvReadfile(sqlite3_file *pFd, void *aBuf, int iAmt, sqlite3_int64 iOff);
sqlite3 *bcvOpenAndInitDb(int *pRc, const char *zDir, char **pzErr);
char *bcvGetFullpath(int *pRc, const char *zDir);
int bcvfsCacheInit(BcvCommon *p, sqlite3 *db, char *zFullDir);
Container *bcvContainerAlloc(
int*, BcvCommon*,
const char *zStorage,
const char *zAccount,
const char *zContainer,
const char *zAlias
);
Container *bcvfsFindContAlias(BcvCommon*, const char*, char **pzErr);
void bcvCommonDestroy(BcvCommon *p);
void bcvContainerFree(Container *pCont);
int bcvContainerDetachAndFree(BcvCommon *p, Container *pCont);
int bcvManifestInstall(BcvCommon *p, Container *pCont, Manifest *pMan);
int bcvCountCached(BcvCommon*, Manifest*, ManifestDb*, int*);
ManifestDb *bcvfsFindDatabase(Manifest *pMan, const char *zDb, int nDb);
CacheEntry *bcvfsHashFind(BcvCommon *p, const u8 *pBlk, int nBlk);
CacheEntry *bcvfsAllocCacheEntry(int *pRc, BcvCommon *p);
void bcvfsHashAdd(BcvCommon *p, CacheEntry *pEntry);
void bcvfsHashRemove(BcvCommon *p, CacheEntry *pEntry);
void bcvfsLruRemoveIf(BcvCommon *p, CacheEntry *pEntry);
void bcvfsEntryUnref(BcvCommon *p, CacheEntry *pEntry);
void bcvfsBlockidToText(const u8 *pBlk, int nBlk, char *aBuf);
ManifestDb *bcvManifestDbidToDb(Manifest *p, i64 iDbId);
void bcvfsLruAdd(BcvCommon *p, CacheEntry *pEntry);
u8 *bcvDatabaseVtabData(
int*, BcvCommon*, const char*, const char*, const char*,
void(*xClientCount)(BcvCommon*,Container*,int,int*,int*,int*),
u32, int*, int*
);
int bcvManifestUpdate(BcvCommon*, Container*, Manifest*, char**);
void bcvExecPrintf(int *pRc, sqlite3 *db, const char *zFmt, ...);
void bcvfsUnusedAdd(BcvCommon *p, CacheEntry *pEntry);
void bcvfsLruAddIf(BcvCommon *p, CacheEntry *pEntry);
int bcvfsNameToBlockid(Manifest *p, const char *zName, u8 *aBlk);
u8 *bcvEmptyKV(int *pRc, int *pnData);
int bcvfsCreateLocalDb(BcvCommon*, const char*, const char*, sqlite3_file**);
void fatal_oom_error(void);
void hex_encode(const unsigned char *aIn, int nIn, char *aBuf, int bUpper);
int hex_decode(const char *aIn, int nIn, u8 *aBuf);
int bcvInstallBuiltinModules(void);
struct BcvContainer {
sqlite3_bcv_container *pCont;
sqlite3_bcv_module *pMod;
int nContRef;
int nMaxResults;
int bRevokeBestEffort;
char *zCainfo;
};
int bcvDispatchNew(BcvDispatch**);
int bcvDispatchRun(BcvDispatch*, struct curl_waitfd *aFd, int nFd, int ms);
int bcvDispatchRunAll(BcvDispatch*);
void bcvDispatchFree(BcvDispatch*);
void bcvDispatchVerbose(BcvDispatch*, int);
void bcvDispatchTimeout(BcvDispatch*, int);
void bcvDispatchLog(BcvDispatch*, void*, void (*xLog)(void*,int,const char*));
void bcvDispatchLogObj(BcvDispatch*, BcvLog*);
int bcvDispatchFetch(
BcvDispatch *p,
BcvContainer *pCont,
const char *zFile,
const char *zETag,
const void *pMd5,
void *pApp,
void (*x)(void*, int rc, char *zETag, const u8*, int, const u8*, int)
);
int bcvDispatchPut(
BcvDispatch *p,
BcvContainer *pCont,
const char *zFile,
const char *zIfMatch,
const u8 *aData, int nData,
void *pApp,
void (*x)(void*, int rc, char *zETag)
);
int bcvDispatchDelete(
BcvDispatch *p,
BcvContainer *pCont,
const char *zFile,
const char *zIfMatch,
void *pApp,
void (*x)(void*, int rc, char *zErr)
);
int bcvDispatchCreate(
BcvDispatch *p,
BcvContainer *pCont,
void *pApp,
void (*x)(void*, int rc, char *zError)
);
int bcvDispatchDestroy(
BcvDispatch *p,
BcvContainer *pCont,
void *pApp,
void (*x)(void*, int rc, char *zError)
);
int bcvDispatchList(
BcvDispatch *p,
BcvContainer *pCont,
const char *zPrefix,
void *pApp,
int (*x)(void*, int rc, char *zError)
);
int bcvDispatchLogmsg(BcvDispatch *p, const char *zFmt, ...);
int bcvDispatchClientId(BcvDispatch *p, const char *z);
int bcvCreateModuleUnsafe(const char*, sqlite3_bcv_module*, void*);
int bcvContainerOpen(
const char *zMod,
const char *zUser,
const char *zAuth,
const char *zCont,
BcvContainer **ppCont,
char **pzErr
);
void bcvContainerClose(BcvContainer*);
int bcv_socket_is_valid(BCV_SOCKET_TYPE s);
void bcv_close_socket(BCV_SOCKET_TYPE s);
void bcv_socket_init();
void sqlite3_bcv_test_socket_api(
int (*xRecv)(BCV_SOCKET_TYPE, void *, int),
int (*xSend)(BCV_SOCKET_TYPE, void *, int),
int (**pxRecv)(BCV_SOCKET_TYPE, void *, int),
int (**pxSend)(BCV_SOCKET_TYPE, void *, int)
);
typedef struct BcvMessage BcvMessage;
typedef struct BcvAttachMsg BcvAttachMsg;
typedef struct BcvCmdMsg BcvCmdMsg;
typedef struct BcvDetachMsg BcvDetachMsg;
typedef struct BcvEndMsg BcvEndMsg;
typedef struct BcvHelloMsg BcvHelloMsg;
typedef struct BcvPassMsg BcvPassMsg;
typedef struct BcvReadMsg BcvReadMsg;
typedef struct BcvVtabMsg BcvVtabMsg;
typedef struct BcvReply BcvReply;
typedef struct BcvHelloReply BcvHelloReply;
typedef struct BcvPassReply BcvPassReply;
typedef struct BcvReadReply BcvReadReply;
typedef struct BcvVtabReply BcvVtabReply;
typedef struct BcvPrefetchMsg BcvPrefetchMsg;
typedef struct BcvPrefetchReply BcvPrefetchReply;
struct BcvHelloMsg {
const char *zContainer;
const char *zDatabase;
int bPrefetch;
};
struct BcvHelloReply {
u32 errCode;
const char *zErrMsg;
const char *zStorage;
const char *zAccount;
const char *zContainer;
i64 iDbId;
u32 szBlk;
u32 bEncrypted;
};
struct BcvPassMsg {
const char *zAuth;
};
struct BcvPassReply {
u32 errCode;
const char *zErrMsg;
u8 *aKey;
};
struct BcvAttachMsg {
const char *zStorage;
const char *zAccount;
const char *zContainer;
const char *zAlias;
const char *zAuth;
u32 flags;
};
struct BcvReply {
u32 errCode;
int szBlk;
const char *zErrMsg;
};
struct BcvVtabMsg {
const char *zVtab;
const char *zContainer;
const char *zDatabase;
u32 colUsed;
int iVersion;
};
struct BcvVtabReply {
int nData;
const u8 *aData;
int iVersion;
};
struct BcvDetachMsg {
const char *zName;
};
struct BcvReadMsg {
u32 iBlk;
const char *zAuth;
u32 nMru;
u32 *aMru;
};
struct BcvReadReply {
u32 errCode;
const char *zErrMsg;
u32 nBlk;
u32 *aBlk;
};
struct BcvEndMsg {
u32 nMru;
u32 *aMru;
};
struct BcvCmdMsg {
const char *zAuth;
u32 eCmd;
};
struct BcvPrefetchMsg {
const char *zAuth;
u32 nRequest;
u32 nMs;
};
struct BcvPrefetchReply {
u32 errCode;
const char *zErrMsg;
u32 nOnDemand;
u32 nOutstanding;
};
#define BCV_CMD_POLL 2
#define BCV_CMD_CLIENT 3
struct BcvMessage {
int eType;
union {
BcvAttachMsg attach;
BcvDetachMsg detach;
BcvHelloMsg hello;
BcvHelloReply hello_r;
BcvReply error_r;
BcvVtabMsg vtab;
BcvVtabReply vtab_r;
BcvReadMsg read;
BcvReadReply read_r;
BcvEndMsg end;
BcvCmdMsg cmd;
BcvPassMsg pass;
BcvPassReply pass_r;
BcvPrefetchMsg prefetch;
BcvPrefetchReply prefetch_r;
} u;
};
int bcvRecvMsg(BCV_SOCKET_TYPE fd, BcvMessage **ppMsg);
int bcvSendMsg(BCV_SOCKET_TYPE fd, BcvMessage *pMsg);
#define BCV_MESSAGE_HELLO 0x01
#define BCV_MESSAGE_HELLO_REPLY 0x02
#define BCV_MESSAGE_REPLY 0x03
#define BCV_MESSAGE_ATTACH 0x04
#define BCV_MESSAGE_VTAB 0x05
#define BCV_MESSAGE_VTAB_REPLY 0x06
#define BCV_MESSAGE_DETACH 0x07
#define BCV_MESSAGE_READ 0x08
#define BCV_MESSAGE_READ_REPLY 0x09
#define BCV_MESSAGE_END 0x0A
#define BCV_MESSAGE_CMD 0x0B
#define BCV_MESSAGE_PASS 0x0C
#define BCV_MESSAGE_PASS_REPLY 0x0D
#define BCV_MESSAGE_PREFETCH 0x0E
#define BCV_MESSAGE_PREFETCH_REPLY 0x0F
BcvIntKey *bcvIntEncryptionKeyRef(BcvIntKey*);
BcvIntKey *bcvIntEncryptionKeyNew(const u8 *aKey);
void bcvIntEncryptionKeyFree(BcvIntKey*);
int bcvIntDecrypt(BcvIntKey*, sqlite3_int64, u8*, u8*, int);
int bcvIntEncrypt(BcvIntKey*, sqlite3_int64, u8*, u8*, int);
const char *bcvRequestHeader(const u8 *aHdrs, int nHdrs, const char *zHdr);
BcvLog *bcvLogNew();
void bcvLogDelete(BcvLog*);
#define BCV_REQUEST_BLOCK_DEMAND
#define BCV_REQUEST_BLOCK_PREFETCH
#define BCV_REQUEST_MANIFEST
#define BCV_REQUEST_BCV_KV
int bcvLogRequest(
BcvLog *pLog,
const char *zClientId,
const char *zMsg,
int eMethod,
int nRetry,
const char *zUri,
i64 *piLogId
);
int bcvLogReply(
BcvLog *pLog,
i64 iLogId,
int httpcode,
i64 *piMs
);
int bcvLogGetData(BcvLog *pLog, BcvBuffer *pBuf);
void bcvTimeToString(i64 iTime, char *zBuf);
void bcvLogConfig(BcvLog *pLog, int op, i64 iVal);
int bcvDeleteBlocks(Manifest *pMan, int iDb);