diff --git a/.forgejo/workflows/compile.yaml b/.forgejo/workflows/compile.yaml new file mode 100644 index 0000000..fc35606 --- /dev/null +++ b/.forgejo/workflows/compile.yaml @@ -0,0 +1,18 @@ +name: Compile Cytoplasm +run-name: Compile Cytoplasm on ${{ forgejo.actor }} +on: [push, pull_request] + +jobs: + "Compile Cytoplasm": + strategy: + matrix: + os: [alpine] + arch: [aarch64] + runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"] + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Configure Cytoplasm + run: ./configure + - name: Build Cytoplasm + run: make diff --git a/.gitea/workflows/compile.yaml b/.gitea/workflows/compile.yaml deleted file mode 100644 index 1185758..0000000 --- a/.gitea/workflows/compile.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Compile Cytoplasm -run-name: Compile Cytoplasm on ${{ gitea.actor }} -on: [push] - -jobs: - "Compile Cytoplasm": - strategy: - matrix: - os: [debian-v12.4, alpine-v3.19, openbsd-v7.4, freebsd-v14.0, netbsd-v9.3] - arch: [x86, x86_64] - exclude: - # 32-bit OpenBSD does not behave well in QEMU. Even when using - # QEMU to emulate i386, it utilizes 100% of its CPU core and is - # still extremely sluggish. Thus, we don't have a working 32-bit - # OpenBSD runner, so exclude it from the matrix configuration. - - os: openbsd-v7.4 - arch: x86 - runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"] - steps: - - name: Check out repository - uses: actions/checkout@v3 - - name: Configure Cytoplasm - run: ./configure - - name: Build Cytoplasm - run: make diff --git a/CHANGELOG.md b/CHANGELOG.md index a646e92..95c7149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ are not even initialized to a default value. - Make `HttpRouter` decode path parts before matching them on regular expressions. - Fixed a bug in `ArraySort()` that would crash programs if the passed array has no elements. +- Adds `Db[OP]Args` functions that are equivalent to their `Db[OP]` counter parts, but + uses an array of string instead of variadic arguments. ## v0.4.0 diff --git a/configure b/configure index 1afdd91..a1331ef 100755 --- a/configure +++ b/configure @@ -15,7 +15,7 @@ TOOLS="tools" # Default compiler flags. These must be supported by all POSIX C compilers. # "Fancy" compilers that have additional options must be detected and set below. -CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}" +CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}"; LIBS="-lm -lpthread" # Default args for all platforms. @@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm" # Set SSL flags depending on the platform. case "$(uname)" in OpenBSD) - SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl" + SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb" ;; *) - SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl" + SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb" ;; esac @@ -37,7 +37,7 @@ case "$(uname)" in # These systems typically use GCC. SCRIPT_ARGS="${SCRIPT_ARGS} --cc=gcc" ;; - OpenBSD|FreeBSD) + OpenBSD|FreeBSD|Darwin) # These systems typically use Clang. SCRIPT_ARGS="${SCRIPT_ARGS} --cc=clang" ;; @@ -80,6 +80,14 @@ for arg in $SCRIPT_ARGS; do TLS_IMPL="" TLS_LIBS="" ;; + --with-lmdb) + EDB_IMPL="EDB_LMDB" + EDB_LIBS="-llmdb" + ;; + --disable-lmdb) + EDB_IMPL="" + EDB_LIBS="" + ;; --prefix=*) PREFIX=$(echo "$arg" | cut -d '=' -f 2-) ;; @@ -104,6 +112,11 @@ if [ -n "$TLS_IMPL" ]; then LIBS="${LIBS} ${TLS_LIBS}" fi +if [ -n "$EDB_IMPL" ]; then + CFLAGS="${CFLAGS} -D${EDB_IMPL}" + LIBS="${LIBS} ${EDB_LIBS}" +fi + CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}" LDFLAGS="${LIBS} ${LDFLAGS}" @@ -140,7 +153,7 @@ print_obj() { get_deps() { src="$1" - ${CC} -I${INCLUDE} -E "$src" \ + ${CC} -I${SRC} -I${INCLUDE} -E "$src" \ | grep '^#' \ | awk '{print $3}' \ | cut -d '"' -f 2 \ @@ -256,8 +269,10 @@ ${TAB}done ${LIB_NAME}: ${OUT}/lib/lib${LIB_NAME}.a ${OUT}/lib/lib${LIB_NAME}.so install: ${LIB_NAME} -${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a -${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so +${TAB}mkdir -p \$(PREFIX)/${OUT}/lib +${TAB}mkdir -p \$(PREFIX)/lib +${TAB}cp ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a +${TAB}cp ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so $(collect ${INCLUDE}/ '' '' \$\(PREFIX\)/include/${LIB_NAME}/ install_out) $(collect ${INCLUDE}/ .h .3 \$\(PREFIX\)/man/man3/${LIB_NAME}- install_man) $(collect ${TOOLS}/ '.c' '' \$\(PREFIX\)/bin/ install_tool) diff --git a/include/Cytoplasm/Cytoplasm.h b/include/Cytoplasm/Cytoplasm.h index 0119e06..a0d6af9 100644 --- a/include/Cytoplasm/Cytoplasm.h +++ b/include/Cytoplasm/Cytoplasm.h @@ -33,7 +33,8 @@ #define CYTOPLASM_VERSION_BETA 0 #define CYTOPLASM_VERSION_STABLE (!CYTOPLASM_VERSION_ALPHA && !CYTOPLASM_VERSION_BETA) -#define STRINGIFY(x) #x +#define XSTRINGIFY(x) #x +#define STRINGIFY(x) XSTRINGIFY(x) /*** * @Nm Cytoplasm diff --git a/include/Cytoplasm/Db.h b/include/Cytoplasm/Db.h index f4c41d2..c481521 100644 --- a/include/Cytoplasm/Db.h +++ b/include/Cytoplasm/Db.h @@ -48,6 +48,17 @@ */ typedef struct Db Db; +/** + * Some "hints" for the database backend for operations. + * Hints are a way for the program to declare what to except of it + * (and the program MUST adhere to these hints, but the backend + * MAY adhere). + */ +typedef enum DbHint { + DB_HINT_READONLY, /* The database reference is treated as read-only */ + DB_HINT_WRITE /* The database reference is treated as read/write */ +} DbHint; + /** * When an object is locked, a reference is returned. This reference * is owned by the current thread, and the database is inaccessible to @@ -68,6 +79,15 @@ typedef struct DbRef DbRef; */ extern Db * DbOpen(char *, size_t); +/** + * Open a LMDB data directory. This function is similar to + * .Fn DbOpen , + * but works with a LMDB-based backend, with the second argument + * being the maximum filesize. This function fails with NULL if ever + * called without LMDB enabled at compiletime. + */ +extern Db * DbOpenLMDB(char *, size_t); + /** * Close the database. This function will flush anything in the cache * to the disk, and then close the data directory. It assumes that @@ -99,6 +119,13 @@ extern void DbMaxCacheSet(Db *, size_t); */ extern DbRef * DbCreate(Db *, size_t,...); +/** + * Behaves like + * .Fn DbCreate , + * but with an array of strings instead of variadic arguments. + */ +extern DbRef * DbCreateArgs(Db *, Array *); + /** * Lock an existing object in the database. This function will fail * if the object does not exist. It takes a variable number of C @@ -109,6 +136,29 @@ extern DbRef * DbCreate(Db *, size_t,...); */ extern DbRef * DbLock(Db *, size_t,...); +/** + * Behaves like + * .Fn DbLock , + * but with an array of strings instead of variadic arguments. + */ +extern DbRef * DbLockArgs(Db *, Array *); + +/** + * Behaves like + * .Fn DbLock , + * but with hints on the reference itself, as + * .Fn DbLock + * itself is read/write. + */ +extern DbRef * DbLockIntent(Db *, DbHint, size_t,...); + +/** + * Behaves like + * .Fn DbLockIntent , + * but with an array of strings instead of variadic arguments. + */ +extern DbRef * DbLockIntentArgs(Db *, DbHint, Array *); + /** * Immediately and permanently remove an object from the database. * This function assumes the object is not locked, otherwise undefined @@ -116,6 +166,13 @@ extern DbRef * DbLock(Db *, size_t,...); */ extern bool DbDelete(Db *, size_t,...); +/** + * Behaves like + * .Fn DbDelete , + * but with an array of strings instead of variadic arguments. + */ +extern bool DbDeleteArgs(Db *, Array *); + /** * Unlock an object and return it back to the database. This function * immediately syncs the object to the filesystem. The cache is a @@ -133,6 +190,13 @@ extern bool DbUnlock(Db *, DbRef *); */ extern bool DbExists(Db *, size_t,...); +/** + * Behaves like + * .Fn DbExists , + * but with an array of strings instead of variadic arguments. + */ +extern bool DbExistsArgs(Db *, Array *); + /** * List all of the objects at a given path. Unlike the other varargs * functions, this one does not take a path to a specific object; it @@ -145,6 +209,13 @@ extern bool DbExists(Db *, size_t,...); */ extern Array * DbList(Db *, size_t,...); +/** + * Behaves like + * .Fn DbList , + * but with an array of strings instead of variadic arguments. + */ +extern Array * DbListArgs(Db *, Array *); + /** * Free the list returned by * .Fn DbListFree . diff --git a/include/Cytoplasm/Json.h b/include/Cytoplasm/Json.h index 261aced..b2a6e23 100644 --- a/include/Cytoplasm/Json.h +++ b/include/Cytoplasm/Json.h @@ -297,6 +297,12 @@ extern size_t JsonEncode(HashMap *, Stream *, int); */ extern HashMap * JsonDecode(Stream *); +/** + * Decodes a JSON value from thr current input strram and parse it + * into a JsonValue. + */ +extern JsonValue * JsonValueDecode(Stream *); + /** * A convenience function that allows the caller to retrieve and * arbitrarily deep keys within a JSON object. It takes a root JSON diff --git a/include/Cytoplasm/Platform.h b/include/Cytoplasm/Platform.h new file mode 100644 index 0000000..c18e7d7 --- /dev/null +++ b/include/Cytoplasm/Platform.h @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CYTOPLASM_PLATFORM_H +#define CYTOPLASM_PLATFORM_H + +/*** + * @Nm Platform + * @Nd A simple macro header that determines what platform the application + * is being built for. + * @Dd September 21, 2024 + */ + +#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__) + #define PLATFORM_WINDOWS + + #ifdef _WIN64 + #define PLATFORM_WIN64 + #else + #define PLATFORM_WIN32 + #endif +#elif __APPLE__ + #define PLATFORM_DARWIN + + #include + #if TARGET_IPHONE_SIMULATOR + #define PLATFORM_IPHONE + #elif TARGET_OS_MACCATALYST + #define PLATFORM_CATALYST + #elif TARGET_OS_IPHONE + #define PLATFORM_IPHONE + #elif TARGET_OS_MAC + #define PLATFORM_MAC + #else + # error "Unknown Apple platform" + #endif +#elif __ANDROID__ + #define PLATFORM_ANDROID +#elif __linux__ + #define PLATFORM_LINUX +#elif __unix__ + #define PLATFORM_UNIX +#elif defined(_POSIX_VERSION) + #define PLATFORM_POSIX +#else +# error "Unknown compiler" +#endif + +#endif /* CYTOPLASM_PLATFORM_H */ diff --git a/include/Cytoplasm/Sha.h b/include/Cytoplasm/Sha.h index a23ef35..8a94fec 100644 --- a/include/Cytoplasm/Sha.h +++ b/include/Cytoplasm/Sha.h @@ -41,6 +41,16 @@ * due to the lack of a 64-bit integer type, so that hash * function has been omitted. */ +#include + +/** + * This is an enum to be used to identify the type of SHA used. + */ +typedef enum HashType +{ + HASH_SHA1, + HASH_SHA256 +} HashType; /** * This function takes a pointer to a NULL-terminated C string, and @@ -64,6 +74,20 @@ extern unsigned char * Sha256(char *); */ extern unsigned char * Sha1(char *); +/** + * This function behaves just like + * .Fn Sha256 , + * except that it allows for a generic byte array, instead of a string. + */ +extern unsigned char * Sha256Raw(unsigned char *, size_t); + +/** + * This function behaves just like + * .Fn Sha1 , + * except that it allows for a generic byte array, instead of a string. + */ +extern unsigned char * Sha1Raw(unsigned char *, size_t); + /** * Convert a SHA byte buffer into a hex string. These hex strings * are typically what is transmitted, stored, and compared, however @@ -71,6 +95,6 @@ extern unsigned char * Sha1(char *); * bytes directly, which is why the conversion to a hex string is * a separate step. */ -extern char * ShaToHex(unsigned char *); +extern char * ShaToHex(unsigned char *, HashType); #endif /* CYTOPLASM_SHA_H */ diff --git a/src/Db.c b/src/Db.c deleted file mode 100644 index 22456ac..0000000 --- a/src/Db.c +++ /dev/null @@ -1,968 +0,0 @@ -/* - * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation files - * (the "Software"), to deal in the Software without restriction, - * including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -struct Db -{ - char *dir; - pthread_mutex_t lock; - - size_t cacheSize; - size_t maxCache; - HashMap *cache; - - /* - * The cache uses a double linked list (see DbRef - * below) to know which objects are most and least - * recently used. The following diagram helps me - * know what way all the pointers go, because it - * can get very confusing sometimes. For example, - * there's nothing stopping "next" from pointing to - * least recent, and "prev" from pointing to most - * recent, so hopefully this clarifies the pointer - * terminology used when dealing with the linked - * list: - * - * mostRecent leastRecent - * | prev prev | prev - * +---+ ---> +---+ ---> +---+ ---> NULL - * |ref| |ref| |ref| - * NULL <--- +---+ <--- +---+ <--- +---+ - * next next next - */ - DbRef *mostRecent; - DbRef *leastRecent; -}; - -struct DbRef -{ - HashMap *json; - - uint64_t ts; - size_t size; - - Array *name; - - DbRef *prev; - DbRef *next; - - int fd; - Stream *stream; -}; - -static void -StringArrayFree(Array * arr) -{ - size_t i; - - for (i = 0; i < ArraySize(arr); i++) - { - Free(ArrayGet(arr, i)); - } - - ArrayFree(arr); -} - -static ssize_t DbComputeSize(HashMap *); - -static ssize_t -DbComputeSizeOfValue(JsonValue * val) -{ - MemoryInfo *a; - ssize_t total = 0; - - size_t i; - - union - { - char *str; - Array *arr; - } u; - - if (!val) - { - return -1; - } - - a = MemoryInfoGet(val); - if (a) - { - total += MemoryInfoGetSize(a); - } - - switch (JsonValueType(val)) - { - case JSON_OBJECT: - total += DbComputeSize(JsonValueAsObject(val)); - break; - case JSON_ARRAY: - u.arr = JsonValueAsArray(val); - a = MemoryInfoGet(u.arr); - - if (a) - { - total += MemoryInfoGetSize(a); - } - - for (i = 0; i < ArraySize(u.arr); i++) - { - total += DbComputeSizeOfValue(ArrayGet(u.arr, i)); - } - break; - case JSON_STRING: - u.str = JsonValueAsString(val); - a = MemoryInfoGet(u.str); - if (a) - { - total += MemoryInfoGetSize(a); - } - break; - case JSON_NULL: - case JSON_INTEGER: - case JSON_FLOAT: - case JSON_BOOLEAN: - default: - /* These don't use any extra heap space */ - break; - } - return total; -} - -static ssize_t -DbComputeSize(HashMap * json) -{ - char *key; - JsonValue *val; - MemoryInfo *a; - size_t total; - - if (!json) - { - return -1; - } - - total = 0; - - a = MemoryInfoGet(json); - if (a) - { - total += MemoryInfoGetSize(a); - } - - while (HashMapIterate(json, &key, (void **) &val)) - { - a = MemoryInfoGet(key); - if (a) - { - total += MemoryInfoGetSize(a); - } - - total += DbComputeSizeOfValue(val); - } - - return total; -} - -static char * -DbHashKey(Array * args) -{ - size_t i; - char *str = NULL; - - for (i = 0; i < ArraySize(args); i++) - { - char *tmp = StrConcat(2, str, ArrayGet(args, i)); - - Free(str); - str = tmp; - } - - return str; -} - -static char * -DbDirName(Db * db, Array * args, size_t strip) -{ - size_t i; - char *str = StrConcat(2, db->dir, "/"); - - for (i = 0; i < ArraySize(args) - strip; i++) - { - char *tmp; - - tmp = StrConcat(3, str, ArrayGet(args, i), "/"); - - Free(str); - - str = tmp; - } - - return str; -} - -static char * -DbFileName(Db * db, Array * args) -{ - size_t i; - char *str = StrConcat(2, db->dir, "/"); - - for (i = 0; i < ArraySize(args); i++) - { - char *tmp; - char *arg = StrDuplicate(ArrayGet(args, i)); - size_t j = 0; - - /* Sanitize name to prevent directory traversal attacks */ - while (arg[j]) - { - switch (arg[j]) - { - case '/': - arg[j] = '_'; - break; - case '.': - arg[j] = '-'; - break; - default: - break; - } - j++; - } - - tmp = StrConcat(3, str, arg, - (i < ArraySize(args) - 1) ? "/" : ".json"); - - Free(arg); - Free(str); - - str = tmp; - } - - return str; -} - -static void -DbCacheEvict(Db * db) -{ - DbRef *ref = db->leastRecent; - DbRef *tmp; - - while (ref && db->cacheSize > db->maxCache) - { - char *hash; - - JsonFree(ref->json); - - hash = DbHashKey(ref->name); - HashMapDelete(db->cache, hash); - Free(hash); - - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - tmp = ref->next; - Free(ref); - - ref = tmp; - } -} - -Db * -DbOpen(char *dir, size_t cache) -{ - Db *db; - pthread_mutexattr_t attr; - - if (!dir) - { - return NULL; - } - - db = Malloc(sizeof(Db)); - if (!db) - { - return NULL; - } - - db->dir = dir; - db->maxCache = cache; - - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&db->lock, &attr); - pthread_mutexattr_destroy(&attr); - - db->mostRecent = NULL; - db->leastRecent = NULL; - db->cacheSize = 0; - - if (db->maxCache) - { - db->cache = HashMapCreate(); - if (!db->cache) - { - return NULL; - } - } - else - { - db->cache = NULL; - } - - return db; -} - -void -DbMaxCacheSet(Db * db, size_t cache) -{ - if (!db) - { - return; - } - - pthread_mutex_lock(&db->lock); - - db->maxCache = cache; - if (db->maxCache && !db->cache) - { - db->cache = HashMapCreate(); - db->cacheSize = 0; - } - - DbCacheEvict(db); - - pthread_mutex_unlock(&db->lock); -} - -void -DbClose(Db * db) -{ - if (!db) - { - return; - } - - pthread_mutex_lock(&db->lock); - - DbMaxCacheSet(db, 0); - DbCacheEvict(db); - HashMapFree(db->cache); - - pthread_mutex_unlock(&db->lock); - pthread_mutex_destroy(&db->lock); - - Free(db); -} - -static DbRef * -DbLockFromArr(Db * db, Array * args) -{ - char *file; - char *hash; - DbRef *ref; - struct flock lock; - - int fd; - Stream *stream; - - if (!db || !args) - { - return NULL; - } - - ref = NULL; - hash = NULL; - - pthread_mutex_lock(&db->lock); - - /* Check if the item is in the cache */ - hash = DbHashKey(args); - ref = HashMapGet(db->cache, hash); - file = DbFileName(db, args); - - fd = open(file, O_RDWR); - if (fd == -1) - { - if (ref) - { - HashMapDelete(db->cache, hash); - JsonFree(ref->json); - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - if (!db->leastRecent) - { - db->leastRecent = db->mostRecent; - } - - Free(ref); - } - ref = NULL; - goto finish; - } - - stream = StreamFd(fd); - - lock.l_start = 0; - lock.l_len = 0; - lock.l_type = F_WRLCK; - lock.l_whence = SEEK_SET; - - /* Lock the file on the disk */ - if (fcntl(fd, F_SETLK, &lock) < 0) - { - StreamClose(stream); - ref = NULL; - goto finish; - } - - if (ref) /* In cache */ - { - uint64_t diskTs = UtilLastModified(file); - - ref->fd = fd; - ref->stream = stream; - - if (diskTs > ref->ts) - { - /* File was modified on disk since it was cached */ - HashMap *json = JsonDecode(ref->stream); - - if (!json) - { - StreamClose(ref->stream); - ref = NULL; - goto finish; - } - - JsonFree(ref->json); - ref->json = json; - ref->ts = diskTs; - ref->size = DbComputeSize(ref->json); - } - - /* Float this ref to mostRecent */ - if (ref->next) - { - ref->next->prev = ref->prev; - - if (!ref->prev) - { - db->leastRecent = ref->next; - } - else - { - ref->prev->next = ref->next; - } - - ref->prev = db->mostRecent; - ref->next = NULL; - if (db->mostRecent) - { - db->mostRecent->next = ref; - } - db->mostRecent = ref; - } - - /* If there is no least recent, this is the only thing in the - * cache, so it is also least recent. */ - if (!db->leastRecent) - { - db->leastRecent = ref; - } - - /* The file on disk may be larger than what we have in memory, - * which may require items in cache to be evicted. */ - DbCacheEvict(db); - } - else - { - Array *name; - size_t i; - - /* Not in cache; load from disk */ - - ref = Malloc(sizeof(DbRef)); - if (!ref) - { - StreamClose(stream); - goto finish; - } - - ref->json = JsonDecode(stream); - - if (!ref->json) - { - Free(ref); - StreamClose(stream); - ref = NULL; - goto finish; - } - - ref->fd = fd; - ref->stream = stream; - - name = ArrayCreate(); - for (i = 0; i < ArraySize(args); i++) - { - ArrayAdd(name, StrDuplicate(ArrayGet(args, i))); - } - ref->name = name; - - if (db->cache) - { - ref->ts = UtilTsMillis(); - ref->size = DbComputeSize(ref->json); - HashMapSet(db->cache, hash, ref); - db->cacheSize += ref->size; - - ref->next = NULL; - ref->prev = db->mostRecent; - if (db->mostRecent) - { - db->mostRecent->next = ref; - } - db->mostRecent = ref; - - if (!db->leastRecent) - { - db->leastRecent = ref; - } - - /* Adding this item to the cache may case it to grow too - * large, requiring some items to be evicted */ - DbCacheEvict(db); - } - } - -finish: - if (!ref) - { - pthread_mutex_unlock(&db->lock); - } - - Free(file); - Free(hash); - - return ref; -} - -DbRef * -DbCreate(Db * db, size_t nArgs,...) -{ - Stream *fp; - char *file; - char *dir; - va_list ap; - Array *args; - DbRef *ret; - - if (!db) - { - return NULL; - } - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return NULL; - } - - pthread_mutex_lock(&db->lock); - - file = DbFileName(db, args); - - if (UtilLastModified(file)) - { - Free(file); - ArrayFree(args); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - dir = DbDirName(db, args, 1); - if (UtilMkdir(dir, 0750) < 0) - { - Free(file); - ArrayFree(args); - Free(dir); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - Free(dir); - - fp = StreamOpen(file, "w"); - Free(file); - if (!fp) - { - ArrayFree(args); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - StreamPuts(fp, "{}"); - StreamClose(fp); - - /* DbLockFromArr() will lock again for us */ - pthread_mutex_unlock(&db->lock); - - ret = DbLockFromArr(db, args); - - ArrayFree(args); - - return ret; -} - -bool -DbDelete(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - char *file; - char *hash; - bool ret = true; - DbRef *ref; - - if (!db) - { - return false; - } - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - pthread_mutex_lock(&db->lock); - - hash = DbHashKey(args); - file = DbFileName(db, args); - - ref = HashMapGet(db->cache, hash); - if (ref) - { - HashMapDelete(db->cache, hash); - JsonFree(ref->json); - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - if (!db->leastRecent) - { - db->leastRecent = db->mostRecent; - } - - Free(ref); - } - - Free(hash); - - if (UtilLastModified(file)) - { - ret = (remove(file) == 0); - } - - pthread_mutex_unlock(&db->lock); - - ArrayFree(args); - Free(file); - return ret; -} - -DbRef * -DbLock(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - DbRef *ret; - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return NULL; - } - - ret = DbLockFromArr(db, args); - - ArrayFree(args); - - return ret; -} - -bool -DbUnlock(Db * db, DbRef * ref) -{ - bool destroy; - - if (!db || !ref) - { - return false; - } - - lseek(ref->fd, 0L, SEEK_SET); - if (ftruncate(ref->fd, 0) < 0) - { - pthread_mutex_unlock(&db->lock); - Log(LOG_ERR, "Failed to truncate file on disk."); - Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno)); - return false; - } - - JsonEncode(ref->json, ref->stream, JSON_DEFAULT); - StreamClose(ref->stream); - - if (db->cache) - { - char *key = DbHashKey(ref->name); - - if (HashMapGet(db->cache, key)) - { - db->cacheSize -= ref->size; - ref->size = DbComputeSize(ref->json); - db->cacheSize += ref->size; - - /* If this ref has grown significantly since we last - * computed its size, it may have filled the cache and - * require some items to be evicted. */ - DbCacheEvict(db); - - destroy = false; - } - else - { - destroy = true; - } - - Free(key); - } - else - { - destroy = true; - } - - if (destroy) - { - JsonFree(ref->json); - StringArrayFree(ref->name); - - Free(ref); - } - - pthread_mutex_unlock(&db->lock); - return true; -} - -bool -DbExists(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - char *file; - bool ret; - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return false; - } - - pthread_mutex_lock(&db->lock); - - file = DbFileName(db, args); - ret = (UtilLastModified(file) != 0); - - pthread_mutex_unlock(&db->lock); - - Free(file); - ArrayFree(args); - - return ret; -} - -Array * -DbList(Db * db, size_t nArgs,...) -{ - Array *result; - Array *path; - DIR *files; - struct dirent *file; - char *dir; - va_list ap; - - if (!db || !nArgs) - { - return NULL; - } - - result = ArrayCreate(); - if (!result) - { - return NULL; - } - - va_start(ap, nArgs); - path = ArrayFromVarArgs(nArgs, ap); - dir = DbDirName(db, path, 0); - - pthread_mutex_lock(&db->lock); - - files = opendir(dir); - if (!files) - { - ArrayFree(path); - ArrayFree(result); - Free(dir); - pthread_mutex_unlock(&db->lock); - return NULL; - } - while ((file = readdir(files))) - { - size_t namlen = strlen(file->d_name); - - if (namlen > 5) - { - int nameOffset = namlen - 5; - - if (StrEquals(file->d_name + nameOffset, ".json")) - { - file->d_name[nameOffset] = '\0'; - ArrayAdd(result, StrDuplicate(file->d_name)); - } - } - } - closedir(files); - - ArrayFree(path); - Free(dir); - pthread_mutex_unlock(&db->lock); - - return result; -} - -void -DbListFree(Array * arr) -{ - StringArrayFree(arr); -} - -HashMap * -DbJson(DbRef * ref) -{ - return ref ? ref->json : NULL; -} - -bool -DbJsonSet(DbRef * ref, HashMap * json) -{ - if (!ref || !json) - { - return false; - } - - JsonFree(ref->json); - ref->json = JsonDuplicate(json); - return true; -} diff --git a/src/Db/Db.c b/src/Db/Db.c new file mode 100644 index 0000000..b275b84 --- /dev/null +++ b/src/Db/Db.c @@ -0,0 +1,562 @@ +/* + * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "Db/Internal.h" + +void +StringArrayFree(Array * arr) +{ + size_t i; + + for (i = 0; i < ArraySize(arr); i++) + { + Free(ArrayGet(arr, i)); + } + + ArrayFree(arr); +} + +static ssize_t DbComputeSize(HashMap *); + +static ssize_t +DbComputeSizeOfValue(JsonValue * val) +{ + MemoryInfo *a; + ssize_t total = 0; + + size_t i; + + union + { + char *str; + Array *arr; + } u; + + if (!val) + { + return -1; + } + + a = MemoryInfoGet(val); + if (a) + { + total += MemoryInfoGetSize(a); + } + + switch (JsonValueType(val)) + { + case JSON_OBJECT: + total += DbComputeSize(JsonValueAsObject(val)); + break; + case JSON_ARRAY: + u.arr = JsonValueAsArray(val); + a = MemoryInfoGet(u.arr); + + if (a) + { + total += MemoryInfoGetSize(a); + } + + for (i = 0; i < ArraySize(u.arr); i++) + { + total += DbComputeSizeOfValue(ArrayGet(u.arr, i)); + } + break; + case JSON_STRING: + u.str = JsonValueAsString(val); + a = MemoryInfoGet(u.str); + if (a) + { + total += MemoryInfoGetSize(a); + } + break; + case JSON_NULL: + case JSON_INTEGER: + case JSON_FLOAT: + case JSON_BOOLEAN: + default: + /* These don't use any extra heap space */ + break; + } + return total; +} + +static ssize_t +DbComputeSize(HashMap * json) +{ + char *key; + JsonValue *val; + MemoryInfo *a; + size_t total; + + if (!json) + { + return -1; + } + + total = 0; + + a = MemoryInfoGet(json); + if (a) + { + total += MemoryInfoGetSize(a); + } + + while (HashMapIterate(json, &key, (void **) &val)) + { + a = MemoryInfoGet(key); + if (a) + { + total += MemoryInfoGetSize(a); + } + + total += DbComputeSizeOfValue(val); + } + + return total; +} + +static char * +DbHashKey(Array * args) +{ + size_t i; + char *str = NULL; + + for (i = 0; i < ArraySize(args); i++) + { + char *tmp = StrConcat(2, str, ArrayGet(args, i)); + + Free(str); + str = tmp; + } + + return str; +} + +static void +DbCacheEvict(Db * db) +{ + DbRef *ref = db->leastRecent; + DbRef *tmp; + + while (ref && db->cacheSize > db->maxCache) + { + char *hash; + + JsonFree(ref->json); + + hash = DbHashKey(ref->name); + HashMapDelete(db->cache, hash); + Free(hash); + + StringArrayFree(ref->name); + + db->cacheSize -= ref->size; + + if (ref->next) + { + ref->next->prev = ref->prev; + } + else + { + db->mostRecent = ref->prev; + } + + if (ref->prev) + { + ref->prev->next = ref->next; + } + else + { + db->leastRecent = ref->next; + } + + tmp = ref->next; + Free(ref); + + ref = tmp; + } +} + +void +DbMaxCacheSet(Db * db, size_t cache) +{ + if (!db) + { + return; + } + + pthread_mutex_lock(&db->lock); + + db->maxCache = cache; + if (db->maxCache && !db->cache) + { + db->cache = HashMapCreate(); + db->cacheSize = 0; + } + + DbCacheEvict(db); + + pthread_mutex_unlock(&db->lock); +} + +void +DbClose(Db * db) +{ + if (!db) + { + return; + } + + pthread_mutex_lock(&db->lock); + if (db->close) + { + db->close(db); + } + DbMaxCacheSet(db, 0); + DbCacheEvict(db); + HashMapFree(db->cache); + + pthread_mutex_unlock(&db->lock); + pthread_mutex_destroy(&db->lock); + + Free(db); +} + +DbRef * +DbCreateArgs(Db * db, Array *args) +{ + if (!args || !db || !db->create) + { + return NULL; + } + + return db->create(db, args); +} +DbRef * +DbCreate(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + if (!db || !db->create) + { + return NULL; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args) + { + return NULL; + } + + ret = DbCreateArgs(db, args); + ArrayFree(args); + + return ret; +} + +bool +DbDeleteArgs(Db * db, Array *args) +{ + if (!args || !db || !db->delete) + { + return false; + } + + return db->delete(db, args); +} +bool +DbDelete(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + bool ret = true; + + if (!db) + { + return false; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + ret = DbDeleteArgs(db, args); + + ArrayFree(args); + return ret; +} + +DbRef * +DbLockArgs(Db * db, Array *args) +{ + if (!args || !db || !db->lockFunc) + { + return NULL; + } + + return db->lockFunc(db, DB_HINT_WRITE, args); +} +DbRef * +DbLock(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + if (!db) + { + return NULL; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args || !db->lockFunc) + { + return NULL; + } + + ret = DbLockArgs(db, args); + + ArrayFree(args); + + return ret; +} + +DbRef * +DbLockIntentArgs(Db * db, DbHint hint, Array *args) +{ + if (!db || !args || !db->lockFunc) + { + return NULL; + } + return db->lockFunc(db, hint, args); +} +DbRef * +DbLockIntent(Db * db, DbHint hint, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args || !db->lockFunc) + { + return NULL; + } + + ret = DbLockIntentArgs(db, hint, args); + + ArrayFree(args); + + return ret; +} + +bool +DbUnlock(Db * db, DbRef * ref) +{ + if (!db || !ref || !db->unlock) + { + return false; + } + + return db->unlock(db, ref); +} + +bool +DbExistsArgs(Db * db, Array *args) +{ + if (!args || !db || !db->exists) + { + return false; + } + + return db->exists(db, args); +} +bool +DbExists(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + bool ret; + if (!db || !nArgs || !db->exists) + { + return false; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args) + { + return false; + } + + ret = DbExistsArgs(db, args); + ArrayFree(args); + + return ret; +} + +Array * +DbListArgs(Db *db, Array *args) +{ + if (!db || !args || !db->list) + { + return NULL; + } + return db->list(db, args); +} +Array * +DbList(Db * db, size_t nArgs,...) +{ + Array *result; + Array *path; + va_list ap; + + if (!db || !nArgs || !db->list) + { + return NULL; + } + + va_start(ap, nArgs); + path = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + result = DbListArgs(db, path); + + ArrayFree(path); + return result; +} + +void +DbListFree(Array * arr) +{ + StringArrayFree(arr); +} + +HashMap * +DbJson(DbRef * ref) +{ + return ref ? ref->json : NULL; +} + +bool +DbJsonSet(DbRef * ref, HashMap * json) +{ + if (!ref || !json) + { + return false; + } + + JsonFree(ref->json); + ref->json = JsonDuplicate(json); + return true; +} +void +DbInit(Db *db) +{ + pthread_mutexattr_t attr; + if (!db) + { + return; + } + + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&db->lock, &attr); + pthread_mutexattr_destroy(&attr); + + db->mostRecent = NULL; + db->leastRecent = NULL; + db->cacheSize = 0; + db->maxCache = 0; + + if (db->maxCache) + { + db->cache = HashMapCreate(); + } + else + { + db->cache = NULL; + } +} +void +DbRefInit(Db *db, DbRef *ref) +{ + if (!db || !ref) + { + return; + } + ref->prev = db->mostRecent; + ref->next = NULL; + ref->json = NULL; + ref->name = NULL; + + /* As default values to be overwritten by impls */ + ref->ts = UtilTsMillis(); + ref->size = 0; + + /* TODO: Append the ref to the cache list. + * I removed it because it broke everything and crashed all the time. + * My bad! */ +} +void +StringArrayAppend(Array *arr, char *str) +{ + if (!arr || !str) + { + return; + } + ArrayAdd(arr, StrDuplicate(str)); +} diff --git a/src/Db/Flat.c b/src/Db/Flat.c new file mode 100644 index 0000000..5945bb4 --- /dev/null +++ b/src/Db/Flat.c @@ -0,0 +1,375 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "Db/Internal.h" + +typedef struct FlatDb { + Db base; + char *dir; + /* Theres not much to do here. */ +} FlatDb; +typedef struct FlatDbRef { + DbRef base; + + Stream *stream; + int fd; +} FlatDbRef; + +static char +DbSanitiseChar(char input) +{ + switch (input) + { + case '/': + return '_'; + case '.': + return '-'; + } + return input; +} + +static char * +DbDirName(FlatDb * db, Array * args, size_t strip) +{ + size_t i, j; + char *str = StrConcat(2, db->dir, "/"); + + for (i = 0; i < ArraySize(args) - strip; i++) + { + char *tmp; + char *sanitise = StrDuplicate(ArrayGet(args, i)); + size_t len = strlen(sanitise); + for (j = 0; j < len; j++) + { + sanitise[j] = DbSanitiseChar(sanitise[j]); + } + + tmp = StrConcat(3, str, sanitise, "/"); + + Free(str); + Free(sanitise); + + str = tmp; + } + + return str; +} +static char * +DbFileName(FlatDb * db, Array * args) +{ + size_t i; + char *str = StrConcat(2, db->dir, "/"); + + for (i = 0; i < ArraySize(args); i++) + { + char *tmp; + char *arg = StrDuplicate(ArrayGet(args, i)); + size_t j = 0; + + /* Sanitize name to prevent directory traversal attacks */ + while (arg[j]) + { + arg[j] = DbSanitiseChar(arg[j]); + j++; + } + + tmp = StrConcat(3, str, arg, + (i < ArraySize(args) - 1) ? "/" : ".json"); + + Free(arg); + Free(str); + + str = tmp; + } + + return str; +} + +static DbRef * +FlatLock(Db *d, DbHint hint, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + FlatDbRef *ref = NULL; + size_t i; + char *path = NULL; + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + path = DbFileName(db, dir); + /* TODO: Caching */ + { + int fd = open(path, O_RDWR); + Stream *stream; + struct flock lock; + if (fd == -1) + { + ref = NULL; + goto end; + } + + stream = StreamFd(fd); + if (!stream) + { + ref = NULL; + goto end; + } + + lock.l_start = 0; + lock.l_len = 0; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + + if (fcntl(fd, F_SETLK, &lock) < 0) + { + StreamClose(stream); + ref = NULL; + goto end; + } + + ref = Malloc(sizeof(*ref)); + DbRefInit(d, (DbRef *) ref); + /* TODO: Hints */ + ref->base.hint = hint; + ref->base.ts = UtilLastModified(path); + ref->base.json = JsonDecode(stream); + ref->stream = stream; + ref->fd = fd; + if (!ref->base.json) + { + Free(ref); + StreamClose(stream); + ref = NULL; + goto end; + } + + + ref->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(dir); i++) + { + StringArrayAppend(ref->base.name, ArrayGet(dir, i)); + } + } +end: + Free(path); + if (!ref) + { + pthread_mutex_unlock(&d->lock); + } + return (DbRef *) ref; +} + +static bool +FlatUnlock(Db *d, DbRef *r) +{ + FlatDbRef *ref = (FlatDbRef *) r; + + if (!d || !r) + { + return false; + } + + lseek(ref->fd, 0L, SEEK_SET); + if (ftruncate(ref->fd, 0) < 0) + { + pthread_mutex_unlock(&d->lock); + Log(LOG_ERR, "Failed to truncate file on disk."); + Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno)); + return false; + } + + JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT); + StreamClose(ref->stream); + + JsonFree(ref->base.json); + StringArrayFree(ref->base.name); + Free(ref); + + pthread_mutex_unlock(&d->lock); + return true; +} +static DbRef * +FlatCreate(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + char *path, *dirPath; + Stream *fp; + DbRef *ret; + + if (!d || !dir) + { + return NULL; + } + pthread_mutex_lock(&d->lock); + + path = DbFileName(db, dir); + if (UtilLastModified(path)) + { + Free(path); + pthread_mutex_unlock(&d->lock); + return NULL; + } + + dirPath = DbDirName(db, dir, 1); + if (UtilMkdir(dirPath, 0750) < 0) + { + Free(path); + Free(dirPath); + pthread_mutex_unlock(&d->lock); + return NULL; + } + Free(dirPath); + + fp = StreamOpen(path, "w"); + Free(path); + if (!fp) + { + pthread_mutex_unlock(&d->lock); + return NULL; + } + + StreamPuts(fp, "{}"); + StreamClose(fp); + + /* FlatLock() will lock again for us */ + pthread_mutex_unlock(&d->lock); + ret = FlatLock(d, DB_HINT_WRITE, dir); + + return ret; +} + +static bool +FlatDelete(Db *d, Array *dir) +{ + bool ret = false; + char *file; + FlatDb *db = (FlatDb *) d; + if (!d || !dir) + { + return false; + } + + pthread_mutex_lock(&d->lock); + file = DbFileName(db, dir); + + /* TODO: Unlink the entry from the linkedlist */ + if (UtilLastModified(file)) + { + ret = remove(file) == 0; + } + + Free(file); + pthread_mutex_unlock(&d->lock); + return ret; +} + +static Array * +FlatList(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + struct dirent *file; + Array *ret; + DIR *files; + char *path; + + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + path = DbDirName(db, dir, 0); + files = opendir(path); + if (!files) + { + Free(path); + pthread_mutex_unlock(&d->lock); + return NULL; + } + + ret = ArrayCreate(); + while ((file = readdir(files))) + { + size_t namlen = strlen(file->d_name); + + if (namlen > 5) + { + int nameOffset = namlen - 5; + + if (StrEquals(file->d_name + nameOffset, ".json")) + { + file->d_name[nameOffset] = '\0'; + StringArrayAppend(ret, file->d_name); + } + } + } + closedir(files); + Free(path); + + pthread_mutex_unlock(&d->lock); + return ret; +} +static bool +FlatExists(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + char *path; + bool ret; + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + path = DbFileName(db, dir); + ret = UtilLastModified(path) != 0; + Free(path); + + pthread_mutex_unlock(&d->lock); + + return ret; +} + +Db * +DbOpen(char *dir, size_t cache) +{ + FlatDb *db; + if (!dir) + { + return NULL; + } + db = Malloc(sizeof(*db)); + DbInit((Db *) db); + db->dir = dir; + db->base.cacheSize = cache; + + db->base.lockFunc = FlatLock; + db->base.unlock = FlatUnlock; + db->base.create = FlatCreate; + db->base.delete = FlatDelete; + db->base.exists = FlatExists; + db->base.list = FlatList; + db->base.close = NULL; + + return (Db *) db; +} diff --git a/src/Db/Internal.h b/src/Db/Internal.h new file mode 100644 index 0000000..1f1014a --- /dev/null +++ b/src/Db/Internal.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CYTOPLASM_DB_INTERNAL_H +#define CYTOPLASM_DB_INTERNAL_H + +#include +#include + +#include + +/* TODO: Define the base structures to define a database internally. + * All "implementations" shall have them as a first element, so that + * basic, general functions can work on them properly. */ + +/* The base structure of a database */ +struct Db +{ + pthread_mutex_t lock; + + size_t cacheSize; + size_t maxCache; + HashMap *cache; + + /* + * The cache uses a double linked list (see DbRef + * below) to know which objects are most and least + * recently used. The following diagram helps me + * know what way all the pointers go, because it + * can get very confusing sometimes. For example, + * there's nothing stopping "next" from pointing to + * least recent, and "prev" from pointing to most + * recent, so hopefully this clarifies the pointer + * terminology used when dealing with the linked + * list: + * + * mostRecent leastRecent + * | prev prev | prev + * +---+ ---> +---+ ---> +---+ ---> NULL + * |ref| |ref| |ref| + * NULL <--- +---+ <--- +---+ <--- +---+ + * next next next + */ + DbRef *mostRecent; + DbRef *leastRecent; + + /* Functions for implementation-specific operations + * (opening a ref, closing a db, removing an entry, ...) */ + DbRef * (*lockFunc)(Db *, DbHint, Array *); + DbRef * (*create)(Db *, Array *); + Array * (*list)(Db *, Array *); + bool (*unlock)(Db *, DbRef *); + bool (*delete)(Db *, Array *); + bool (*exists)(Db *, Array *); + void (*close)(Db *); + + /* Implementation-specific constructs */ +}; + +struct DbRef +{ + HashMap *json; + + uint64_t ts; + size_t size; + + Array *name; + + DbRef *prev; + DbRef *next; + + DbHint hint; + /* Implementation-specific constructs */ +}; + +extern void DbInit(Db *); +extern void DbRefInit(Db *, DbRef *); +extern void StringArrayFree(Array *); +extern void StringArrayAppend(Array *, char *); + +#endif diff --git a/src/Db/LMDB.c b/src/Db/LMDB.c new file mode 100644 index 0000000..7c7eee6 --- /dev/null +++ b/src/Db/LMDB.c @@ -0,0 +1,593 @@ +#include +#include +#include +#include + +#include "Db/Internal.h" + +#include +#include +#include + +#ifdef EDB_LMDB + +#include + +typedef struct LMDB { + Db base; /* The base implementation required to pass */ + + MDB_env *environ; + MDB_dbi dbi; +} LMDB; +typedef struct LMDBRef { + DbRef base; + + /* TODO: LMDB-dependent stuff */ + MDB_txn *transaction; + MDB_dbi dbi; +} LMDBRef; + +/* Helper functions */ +static MDB_val +LMDBTranslateKey(Array *key) +{ + MDB_val ret = { 0 }; + char *data = NULL; + size_t length = 0; + size_t i; + if (!key || ArraySize(key) > 255) + { + return ret; + } + + data = Realloc(data, ++length); + data[0] = ArraySize(key); + + /* Now, let's push every item */ + for (i = 0; i < ArraySize(key); i++) + { + char *entry = ArrayGet(key, i); + size_t offset = length; + + data = Realloc(data, (length += strlen(entry) + 1)); + memcpy(data + offset, entry, strlen(entry)); + data[length - 1] = '\0'; + } + + /* We now have every key */ + ret.mv_size = length; + ret.mv_data = data; + return ret; +} +static void +LMDBKillKey(MDB_val key) +{ + if (!key.mv_data || !key.mv_size) + { + return; + } + + Free(key.mv_data); +} +static HashMap * +LMDBDecode(MDB_val val) +{ + FILE *fakefile; + Stream *fakestream; + HashMap *ret; + if (!val.mv_data || !val.mv_size) + { + return NULL; + } + + fakefile = fmemopen(val.mv_data, val.mv_size, "r"); + fakestream = StreamFile(fakefile); + ret = JsonDecode(fakestream); + StreamClose(fakestream); + + return ret; +} +static bool +LMDBKeyStartsWith(MDB_val key, MDB_val starts) +{ + size_t i; + if (!key.mv_size || !starts.mv_size) + { + return false; + } + if (key.mv_size < starts.mv_size) + { + return false; + } + + for (i = 0; i < starts.mv_size; i++) + { + char keyC = ((char *) key.mv_data)[i]; + char startC = ((char *) starts.mv_data)[i]; + + if (keyC != startC) + { + return false; + } + } + return true; +} +static char * +LMDBKeyHead(MDB_val key) +{ + char *end; + if (!key.mv_size || !key.mv_data) + { + return NULL; + } + /* -2 because we have a NULL byte in there */ + end = ((char *) key.mv_data) + key.mv_size - 1; + if ((void *) end < key.mv_data) + { + /* Uh oh. */ + return NULL; + } + + /* Doing >= will lead to cases where it is sent straight to the + * start. Don't do that. */ + while ((void *) (end - 1) > key.mv_data && *(end - 1)) + { + end--; + } + return end; +} + +static DbRef * +LMDBLock(Db *d, DbHint hint, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + LMDBRef *ret = NULL; + MDB_val key, json_val; + int code, flags; + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* Create a transaction, honoring hints. */ + /* TODO: Do we want to create a "main" transaction that everyone inherits + * from? */ + flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0; + if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + json_val.mv_size = 0; + json_val.mv_data = NULL; + code = mdb_get(transaction, db->dbi, &key, &json_val); + if (code == MDB_NOTFOUND) + { + mdb_txn_abort(transaction); + goto end; + } + else if (code != 0) + { + Log(LOG_ERR, + "%s: mdb_get failure: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(transaction); + goto end; + } + + ret = Malloc(sizeof(*ret)); + DbRefInit(d, (DbRef *) ret); + /* TODO: Timestamp */ + { + size_t i; + ret->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(k); i++) + { + char *ent = ArrayGet(k, i); + StringArrayAppend(ret->base.name, ent); + } + } + ret->base.json = LMDBDecode(json_val); + ret->base.hint = hint; + ret->transaction = NULL; + + if (hint == DB_HINT_WRITE) + { + ret->transaction = transaction; + } + else + { + mdb_txn_abort(transaction); + } +end: + if (!ret || hint == DB_HINT_READONLY) + { + pthread_mutex_unlock(&d->lock); + } + LMDBKillKey(key); + return (DbRef *) ret; +} +static bool +LMDBExists(Db *d, Array *k) +{ + MDB_val key, empty; + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + int code; + bool ret = false; + if (!d || !k) + { + return false; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + ret = mdb_get(transaction, db->dbi, &key, &empty) == 0; + mdb_txn_abort(transaction); +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} +static bool +LMDBDelete(Db *d, Array *k) +{ + MDB_val key, empty; + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + int code; + bool ret = false; + if (!d || !k) + { + return false; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + ret = mdb_del(transaction, db->dbi, &key, &empty) == 0; + mdb_txn_commit(transaction); +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} + +static bool +LMDBUnlock(Db *d, DbRef *r) +{ + LMDBRef *ref = (LMDBRef *) r; + LMDB *db = (LMDB *) d; + FILE *fakestream; + Stream *stream; + MDB_val key, val; + bool ret = true; + DbHint hint = r ? r->hint : 0; + + if (!d || !r) + { + return false; + } + + val.mv_data = NULL; + val.mv_size = 0; + if (ref->transaction && r->hint == DB_HINT_WRITE) + { + key = LMDBTranslateKey(r->name); + + fakestream = open_memstream((char **) &val.mv_data, &val.mv_size); + stream = StreamFile(fakestream); + JsonEncode(r->json, stream, JSON_DEFAULT); + StreamFlush(stream); + StreamClose(stream); + + ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0; + + mdb_txn_commit(ref->transaction); + LMDBKillKey(key); + } + StringArrayFree(ref->base.name); + JsonFree(ref->base.json); + Free(ref); + + if (val.mv_data) + { + free(val.mv_data); + } + if (ret && hint == DB_HINT_WRITE) + { + pthread_mutex_unlock(&d->lock); + } + return ret; +} +static DbRef * +LMDBCreate(Db *d, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + LMDBRef *ret = NULL; + MDB_val key, empty_json; + int code; + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + empty_json.mv_size = 2; + empty_json.mv_data = "{}"; + /* put data in it */ + code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE); + if (code == MDB_KEYEXIST) + { + mdb_txn_abort(transaction); + goto end; + } + else if (code == MDB_MAP_FULL) + { + Log(LOG_ERR, "%s: db is full", __func__); + mdb_txn_abort(transaction); + goto end; + } + else if (code != 0) + { + Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code)); + mdb_txn_abort(transaction); + goto end; + } + + ret = Malloc(sizeof(*ret)); + DbRefInit(d, (DbRef *) ret); + /* TODO: Timestamp */ + { + size_t i; + ret->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(k); i++) + { + char *ent = ArrayGet(k, i); + StringArrayAppend(ret->base.name, ent); + } + } + ret->base.hint = DB_HINT_WRITE; + ret->base.json = HashMapCreate(); + ret->transaction = transaction; +end: + if (!ret) + { + pthread_mutex_unlock(&d->lock); + } + LMDBKillKey(key); + return (DbRef *) ret; +} + +static Array * +LMDBList(Db *d, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_val key = { 0 }; + MDB_val subKey; + MDB_val val; + Array *ret = NULL; + + MDB_cursor *cursor; + MDB_cursor_op op = MDB_SET_RANGE; + MDB_txn *txn; + int code; + + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + /* Marked as read-only, as we just don't need to write anything + * when listing */ + if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0) + { + Log(LOG_ERR, + "%s: could not get cursor: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(txn); + goto end; + } + + key = LMDBTranslateKey(k); + + /* Small hack to get it to list subitems */ + ((char *) key.mv_data)[0]++; + + ret = ArrayCreate(); + subKey = key; + while (mdb_cursor_get(cursor, &subKey, &val, op) == 0) + { + /* This searches by *increasing* order. The problem is that it may + * extend to unwanted points. Since the values are sorted, we can + * just exit if the subkey is incorrect. */ + char *head = LMDBKeyHead(subKey); + if (!LMDBKeyStartsWith(subKey, key)) + { + break; + } + + StringArrayAppend(ret, head); + op = MDB_NEXT; + } + + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} + +/* Implementation functions */ +static void +LMDBClose(Db *d) +{ + LMDB *db = (LMDB *) d; + if (!d) + { + return; + } + + mdb_dbi_close(db->environ, db->dbi); + mdb_env_close(db->environ); +} + +Db * +DbOpenLMDB(char *dir, size_t size) +{ + /* TODO */ + MDB_env *env = NULL; + MDB_txn *txn; + MDB_dbi dbi; + int code; + LMDB *db; + if (!dir || !size) + { + return NULL; + } + + /* Try initialising LMDB */ + if ((code = mdb_env_create(&env)) != 0) + { + Log(LOG_ERR, + "%s: could not create LMDB env: %s", + __func__, mdb_strerror(code) + ); + return NULL; + } + if ((code = mdb_env_set_mapsize(env, size) != 0)) + { + Log(LOG_ERR, + "%s: could not set mapsize to %lu: %s", + __func__, (unsigned long) size, + mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + mdb_env_set_maxdbs(env, 4); + if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0) + { + Log(LOG_ERR, + "%s: could not open LMDB env: %s", + __func__, mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + + /* Initialise a DBI */ + { + if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0) + { + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0) + { + Log(LOG_ERR, + "%s: could not get transaction dbi: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(txn); + mdb_env_close(env); + return NULL; + } + mdb_txn_commit(txn); + } + + + db = Malloc(sizeof(*db)); + DbInit((Db *) db); + db->environ = env; + db->dbi = dbi; + + db->base.lockFunc = LMDBLock; + db->base.create = LMDBCreate; + db->base.unlock = LMDBUnlock; + db->base.delete = LMDBDelete; + db->base.exists = LMDBExists; + db->base.close = LMDBClose; + db->base.list = LMDBList; + + return (Db *) db; +} + +#else +Db * +DbOpenLMDB(char *dir, size_t size) +{ + /* Unimplemented function */ + Log(LOG_ERR, + "LMDB support is not enabled. Please compile with --use-lmdb" + ); + (void) size; + (void) dir; + return NULL; +} +#endif diff --git a/src/HashMap.c b/src/HashMap.c index 32a3199..eca0bdb 100644 --- a/src/HashMap.c +++ b/src/HashMap.c @@ -184,6 +184,8 @@ HashMapDelete(HashMap * map, const char *key) if (bucket->hash == hash) { bucket->hash = 0; + Free(bucket->key); + bucket->key = NULL; return bucket->value; } diff --git a/src/Http.c b/src/Http.c index 3572c66..6bbeacb 100644 --- a/src/Http.c +++ b/src/Http.c @@ -621,7 +621,7 @@ HttpParseHeaders(Stream * fp) strncpy(headerValue, headerPtr, len); - HashMapSet(headers, headerKey, headerValue); + Free(HashMapSet(headers, headerKey, headerValue)); Free(headerKey); } diff --git a/src/Json.c b/src/Json.c index 872c266..df33703 100644 --- a/src/Json.c +++ b/src/Json.c @@ -1348,6 +1348,25 @@ JsonDecode(Stream * stream) return result; } +JsonValue * +JsonValueDecode(Stream *stream) +{ + JsonValue *result; + JsonParserState state; + + state.stream = stream; + state.token = NULL; + + JsonTokenSeek(&state); + result = JsonDecodeValue(&state); + + if (state.token) + { + Free(state.token); + } + + return result; +} JsonValue * JsonGet(HashMap * json, size_t nArgs,...) diff --git a/src/Memory.c b/src/Memory.c index 4083143..e31ec27 100644 --- a/src/Memory.c +++ b/src/Memory.c @@ -40,29 +40,54 @@ #define MEMORY_HEXDUMP_WIDTH 16 #endif +#define MEMORY_FILE_SIZE 256 + +#define MEM_BOUND_TYPE uint64_t +#define MEM_BOUND 0xDEADBEEFBEEFDEAD +#define MEM_MAGIC 0xDEADBEEFDEADBEEF + struct MemoryInfo { + uint64_t magic; + size_t size; - const char *file; + size_t unalignedSize; + char file[MEMORY_FILE_SIZE]; int line; void *pointer; + + MemoryInfo *prev; + MemoryInfo *next; + + MEM_BOUND_TYPE leftBoundary; }; -#define MEM_BOUND_TYPE uint32_t -#define MEM_BOUND 0xDEADBEEF +#define MEM_SIZE_ACTUAL(x) (MemoryAlignBoundary((x) * sizeof(uint8_t)) + sizeof(MEM_BOUND_TYPE)) +#define MEM_START_BOUNDARY(info) (info->leftBoundary) +#define MEM_END_BOUNDARY(info) (*(((MEM_BOUND_TYPE *) (((uint8_t *) info->pointer) + info->size)) - 1)) -#define MEM_BOUND_LOWER(p) *((MEM_BOUND_TYPE *) p) -#define MEM_BOUND_UPPER(p, x) *(((MEM_BOUND_TYPE *) (((uint8_t *) p) + x)) + 1) -#define MEM_SIZE_ACTUAL(x) (((x) * sizeof(uint8_t)) + (2 * sizeof(MEM_BOUND_TYPE))) static pthread_mutex_t lock; static void (*hook) (MemoryAction, MemoryInfo *, void *) = MemoryDefaultHook; static void *hookArgs = NULL; -static MemoryInfo **allocations = NULL; -static size_t allocationsSize = 0; static size_t allocationsLen = 0; +static MemoryInfo *allocationTail = NULL; + +/* Simple range of "plausible" boundaries for heap, serving as an extra + * check */ +static void *heapStart, *heapEnd; + +static size_t MemoryAlignBoundary(size_t size) +{ + size_t boundSize = sizeof(MEM_BOUND_TYPE); + size_t remainder = size % boundSize; + size_t closest = size / boundSize + !!remainder; + + return closest * boundSize; +} + int MemoryRuntimeInit(void) { @@ -77,6 +102,8 @@ MemoryRuntimeInit(void) pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); ret = pthread_mutex_init(&lock, &attr); pthread_mutexattr_destroy(&attr); + heapStart = NULL; + heapEnd = NULL; ret = (ret == 0); @@ -91,70 +118,27 @@ MemoryRuntimeDestroy(void) return pthread_mutex_destroy(&lock) == 0; } -static size_t -MemoryHash(void *p) -{ - return (((size_t) p) >> 2 * 7) % allocationsSize; -} - static int MemoryInsert(MemoryInfo * a) { - size_t hash; - - if (!allocations) + if (allocationTail) { - allocationsSize = MEMORY_TABLE_CHUNK; - allocations = calloc(allocationsSize, sizeof(void *)); - if (!allocations) - { - return 0; - } + allocationTail->next = a; + } + a->next = NULL; + a->prev = allocationTail; + a->magic = MEM_MAGIC; + + if (!heapStart || heapStart > (void *) a) + { + heapStart = a; + } + if (!heapEnd || heapEnd < (void *) a) + { + heapEnd = a; } - /* If the next insertion would cause the table to be at least 3/4 - * full, re-allocate and re-hash. */ - if ((allocationsLen + 1) >= ((allocationsSize * 3) >> 2)) - { - size_t i; - size_t tmpAllocationsSize = allocationsSize; - MemoryInfo **tmpAllocations; - - allocationsSize += MEMORY_TABLE_CHUNK; - tmpAllocations = calloc(allocationsSize, sizeof(void *)); - - if (!tmpAllocations) - { - return 0; - } - - for (i = 0; i < tmpAllocationsSize; i++) - { - if (allocations[i]) - { - hash = MemoryHash(allocations[i]->pointer); - - while (tmpAllocations[hash]) - { - hash = (hash + 1) % allocationsSize; - } - - tmpAllocations[hash] = allocations[i]; - } - } - - free(allocations); - allocations = tmpAllocations; - } - - hash = MemoryHash(a->pointer); - - while (allocations[hash]) - { - hash = (hash + 1) % allocationsSize; - } - - allocations[hash] = a; + allocationTail = a; allocationsLen++; return 1; @@ -163,30 +147,32 @@ MemoryInsert(MemoryInfo * a) static void MemoryDelete(MemoryInfo * a) { - size_t hash = MemoryHash(a->pointer); - size_t count = 0; + MemoryInfo *aPrev = a->prev; + MemoryInfo *aNext = a->next; - while (count <= allocationsSize) + if (aPrev) { - if (allocations[hash] && allocations[hash] == a) - { - allocations[hash] = NULL; - allocationsLen--; - return; - } - else - { - hash = (hash + 1) % allocationsSize; - count++; - } + aPrev->next = aNext; } + if (aNext) + { + aNext->prev = aPrev; + } + + if (a == allocationTail) + { + allocationTail = aPrev; + } + + a->magic = ~MEM_MAGIC; } static int MemoryCheck(MemoryInfo * a) { - if (MEM_BOUND_LOWER(a->pointer) != MEM_BOUND || - MEM_BOUND_UPPER(a->pointer, a->size - (2 * sizeof(MEM_BOUND_TYPE))) != MEM_BOUND) + if (MEM_START_BOUNDARY(a) != MEM_BOUND || + a->magic != MEM_MAGIC || + MEM_END_BOUNDARY(a) != MEM_BOUND) { if (hook) { @@ -203,38 +189,32 @@ MemoryAllocate(size_t size, const char *file, int line) void *p; MemoryInfo *a; - MemoryIterate(NULL, NULL); + //MemoryIterate(NULL, NULL); pthread_mutex_lock(&lock); - a = malloc(sizeof(MemoryInfo)); + a = malloc(sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size)); if (!a) { pthread_mutex_unlock(&lock); return NULL; } - p = malloc(MEM_SIZE_ACTUAL(size)); - if (!p) - { - free(a); - pthread_mutex_unlock(&lock); - return NULL; - } + p = a + 1; memset(p, 0, MEM_SIZE_ACTUAL(size)); - MEM_BOUND_LOWER(p) = MEM_BOUND; - MEM_BOUND_UPPER(p, size) = MEM_BOUND; a->size = MEM_SIZE_ACTUAL(size); - a->file = file; + a->unalignedSize = size; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; a->pointer = p; + MEM_START_BOUNDARY(a) = MEM_BOUND; + MEM_END_BOUNDARY(a) = MEM_BOUND; if (!MemoryInsert(a)) { free(a); - free(p); pthread_mutex_unlock(&lock); return NULL; } @@ -245,7 +225,7 @@ MemoryAllocate(size_t size, const char *file, int line) } pthread_mutex_unlock(&lock); - return ((MEM_BOUND_TYPE *) p) + 1; + return p; } void * @@ -254,7 +234,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) MemoryInfo *a; void *new = NULL; - MemoryIterate(NULL, NULL); + //MemoryIterate(NULL, NULL); if (!p) { @@ -265,25 +245,28 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) if (a) { pthread_mutex_lock(&lock); - new = realloc(a->pointer, MEM_SIZE_ACTUAL(size)); + MemoryDelete(a); + new = realloc(a, sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size)); if (new) { - MemoryDelete(a); + a = new; + a->unalignedSize = size; a->size = MEM_SIZE_ACTUAL(size); - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; + a->magic = MEM_MAGIC; - a->pointer = new; + a->pointer = a + 1; MemoryInsert(a); - MEM_BOUND_LOWER(a->pointer) = MEM_BOUND; - MEM_BOUND_UPPER(a->pointer, size) = MEM_BOUND; + MEM_START_BOUNDARY(a) = MEM_BOUND; + MEM_END_BOUNDARY(a) = MEM_BOUND; if (hook) { hook(MEMORY_REALLOCATE, a, hookArgs); } - + } pthread_mutex_unlock(&lock); } @@ -293,7 +276,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) if (a) { a->size = 0; - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; a->pointer = p; hook(MEMORY_BAD_POINTER, a, hookArgs); @@ -301,7 +284,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) } } - return ((MEM_BOUND_TYPE *) new) + 1; + return a->pointer; } void @@ -309,7 +292,7 @@ MemoryFree(void *p, const char *file, int line) { MemoryInfo *a; - MemoryIterate(NULL, NULL); + //MemoryIterate(NULL, NULL); if (!p) { @@ -322,12 +305,11 @@ MemoryFree(void *p, const char *file, int line) pthread_mutex_lock(&lock); if (hook) { - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; hook(MEMORY_FREE, a, hookArgs); } MemoryDelete(a); - free(a->pointer); free(a); pthread_mutex_unlock(&lock); @@ -337,7 +319,7 @@ MemoryFree(void *p, const char *file, int line) a = malloc(sizeof(MemoryInfo)); if (a) { - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; a->size = 0; a->pointer = p; @@ -350,17 +332,15 @@ MemoryFree(void *p, const char *file, int line) size_t MemoryAllocated(void) { - size_t i; size_t total = 0; + MemoryInfo *cur; pthread_mutex_lock(&lock); - for (i = 0; i < allocationsSize; i++) + /* TODO */ + for (cur = allocationTail; cur; cur = cur->prev) { - if (allocations[i]) - { - total += MemoryInfoGetSize(allocations[i]); - } + total += MemoryInfoGetSize(cur); } pthread_mutex_unlock(&lock); @@ -371,55 +351,44 @@ MemoryAllocated(void) void MemoryFreeAll(void) { - size_t i; + MemoryInfo *cur; + MemoryInfo *prev; pthread_mutex_lock(&lock); - for (i = 0; i < allocationsSize; i++) + /* TODO */ + for (cur = allocationTail; cur; cur = prev) { - if (allocations[i]) - { - free(allocations[i]->pointer); - free(allocations[i]); - } + prev = cur->prev; + free(cur); } - free(allocations); - allocations = NULL; - allocationsSize = 0; allocationsLen = 0; pthread_mutex_unlock(&lock); } MemoryInfo * -MemoryInfoGet(void *p) +MemoryInfoGet(void *po) { - size_t hash, count; + void *p = po; pthread_mutex_lock(&lock); - p = ((MEM_BOUND_TYPE *) p) - 1; - hash = MemoryHash(p); - - count = 0; - while (count <= allocationsSize) + p = ((MemoryInfo *) p) - 1; + if (p < heapStart || p > heapEnd) { - if (!allocations[hash] || allocations[hash]->pointer != p) - { - hash = (hash + 1) % allocationsSize; - count++; - } - else - { - MemoryCheck(allocations[hash]); - pthread_mutex_unlock(&lock); - return allocations[hash]; - } + pthread_mutex_unlock(&lock); + return NULL; + } + + if (((MemoryInfo *)p)->magic != MEM_MAGIC) + { + p = NULL; } pthread_mutex_unlock(&lock); - return NULL; + return p; } size_t @@ -430,7 +399,7 @@ MemoryInfoGetSize(MemoryInfo * a) return 0; } - return a->size ? a->size - (2 * sizeof(MEM_BOUND_TYPE)) : 0; + return a->size ? a->unalignedSize : 0; } const char * @@ -463,25 +432,22 @@ MemoryInfoGetPointer(MemoryInfo * a) return NULL; } - return ((MEM_BOUND_TYPE *) a->pointer) + 1; + return a->pointer; } void - MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args) +MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args) { - size_t i; + MemoryInfo *cur; pthread_mutex_lock(&lock); - for (i = 0; i < allocationsSize; i++) + for (cur = allocationTail; cur; cur = cur->prev) { - if (allocations[i]) + MemoryCheck(cur); + if (iterFunc) { - MemoryCheck(allocations[i]); - if (iterFunc) - { - iterFunc(allocations[i], args); - } + iterFunc(cur, args); } } diff --git a/src/Sha.c b/src/Sha.c index 377ae14..32e2e12 100644 --- a/src/Sha.c +++ b/src/Sha.c @@ -28,21 +28,44 @@ #include char * -ShaToHex(unsigned char *bytes) +ShaToHex(unsigned char *bytes, HashType type) { - size_t i = 0; - char *str = Malloc(((strlen((char *) bytes) * 2) + 1) * sizeof(char)); + size_t i = 0, size; + char *str; + switch (type) + { + case HASH_SHA1: + size = 20; + break; + case HASH_SHA256: + size = 32; + break; + default: + return NULL; + } + + str = Malloc(((size * 2) + 1) * sizeof(char)); if (!str) { return NULL; } - while (bytes[i] != '\0') + for (i = 0; i < size; i++) { snprintf(str + (2 * i), 3, "%02x", bytes[i]); - i++; } return str; } +unsigned char * +Sha256(char *str) +{ + return Sha256Raw((unsigned char *) str, str ? strlen(str) : 0); +} + +unsigned char * +Sha1(char *str) +{ + return Sha1Raw((unsigned char *) str, str ? strlen(str) : 0); +} diff --git a/src/Sha/Sha1.c b/src/Sha/Sha1.c index 9e12d56..c4f9db8 100644 --- a/src/Sha/Sha1.c +++ b/src/Sha/Sha1.c @@ -28,6 +28,27 @@ #include +/* TODO: Verify LibreSSL support later */ +#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL) + +#include + +unsigned char * +Sha1Raw(unsigned char *str, size_t len) +{ + unsigned char *digest; + if (!str) + { + return NULL; + } + + digest = Malloc(20 + 1); + SHA1(str, len, digest); + digest[20] = '\0'; + return digest; +} +#else + #define LOAD32H(x, y) \ { \ x = ((uint32_t)((y)[0] & 255) << 24) | \ @@ -240,7 +261,7 @@ Sha1Calculate(Sha1Context * ctx, unsigned char *out) } unsigned char * -Sha1(char *str) +Sha1Raw(unsigned char *str, size_t len) { Sha1Context ctx; unsigned char *out; @@ -257,10 +278,11 @@ Sha1(char *str) } Sha1Init(&ctx); - Sha1Update(&ctx, str, strlen(str)); + Sha1Update(&ctx, str, len); Sha1Calculate(&ctx, out); out[160 / 8] = '\0'; return out; } +#endif diff --git a/src/Sha/Sha256.c b/src/Sha/Sha256.c index 2563246..0b44b7e 100644 --- a/src/Sha/Sha256.c +++ b/src/Sha/Sha256.c @@ -21,14 +21,36 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include +#include #include #include #include + +/* TODO: Verify LibreSSL support later */ +#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL) + +#include + +unsigned char * +Sha256Raw(unsigned char *str, size_t len) +{ + unsigned char *digest; + if (!str) + { + return NULL; + } + + digest = Malloc(32 + 1); + SHA256(str, len, digest); + digest[32] = '\0'; + return digest; +} +#else + #define GET_UINT32(x) \ (((uint32_t)(x)[0] << 24) | \ ((uint32_t)(x)[1] << 16) | \ @@ -170,7 +192,7 @@ Sha256Process(Sha256Context * context, unsigned char *data, size_t length) } unsigned char * -Sha256(char *str) +Sha256Raw(unsigned char *str, size_t len) { Sha256Context context; size_t i; @@ -206,7 +228,7 @@ Sha256(char *str) context.length = 0; memset(context.buffer, 0, 64); - Sha256Process(&context, (unsigned char *) str, strlen(str)); + Sha256Process(&context, str, len); memset(fill, 0, 64); fill[0] = 0x80; @@ -230,3 +252,4 @@ Sha256(char *str) return out; } +#endif diff --git a/src/Tls/TlsLibreSSL.c b/src/Tls/TlsLibreSSL.c index dc09b11..3f805ae 100644 --- a/src/Tls/TlsLibreSSL.c +++ b/src/Tls/TlsLibreSSL.c @@ -23,7 +23,7 @@ */ #include -#if TLS_IMPL == TLS_LIBRESSL +#if defined(TLS_IMPL) && (TLS_IMPL == TLS_LIBRESSL) #include #include diff --git a/src/Tls/TlsOpenSSL.c b/src/Tls/TlsOpenSSL.c index c056ebc..591c6f4 100644 --- a/src/Tls/TlsOpenSSL.c +++ b/src/Tls/TlsOpenSSL.c @@ -23,7 +23,7 @@ */ #include -#if TLS_IMPL == TLS_OPENSSL +#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL) #include #include @@ -71,14 +71,6 @@ TlsInitClient(int fd, const char *serverName) OpenSSLCookie *cookie; char errorStr[256]; - /* - * TODO: Seems odd that this isn't needed to make the - * connection... we should figure out how to verify the - * certificate matches the server we think we're - * connecting to. - */ - (void) serverName; - cookie = Malloc(sizeof(OpenSSLCookie)); if (!cookie) { @@ -89,12 +81,14 @@ TlsInitClient(int fd, const char *serverName) cookie->method = TLS_client_method(); cookie->ctx = SSL_CTX_new(cookie->method); + cookie->fd = fd; if (!cookie->ctx) { goto error; } cookie->ssl = SSL_new(cookie->ctx); + SSL_set_tlsext_host_name(cookie->ssl, serverName); if (!cookie->ssl) { goto error; @@ -295,9 +289,7 @@ TlsClose(void *cookie) SSL_free(ssl->ssl); SSL_CTX_free(ssl->ctx); -#if 0 close(ssl->fd); -#endif Free(ssl); diff --git a/src/Util.c b/src/Util.c index 962ff0b..e01ae4f 100644 --- a/src/Util.c +++ b/src/Util.c @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -89,6 +90,10 @@ UtilTsMillis(void) return ts; } +#ifdef PLATFORM_DARWIN +#define st_mtim st_mtimespec +#endif + uint64_t UtilLastModified(char *path) { @@ -105,6 +110,10 @@ UtilLastModified(char *path) return ts; } +#ifdef PLATFORM_DARWIN +#undef st_mtim +#endif + int UtilMkdir(const char *dir, const mode_t mode) { diff --git a/tools/j2s.c b/tools/j2s.c index 1ef0660..304741e 100644 --- a/tools/j2s.c +++ b/tools/j2s.c @@ -38,6 +38,7 @@ #define MAX_DEPENDENCIES 32 +#define IsDelimited(field, c1, c2) (*field == c1 && field[strlen(field) - 1] == c2) static char * Trim(char c, char *str) { @@ -75,10 +76,14 @@ TypeToJsonType(char *type) } else { - if (*type == '[' && type[strlen(type) - 1] == ']') + if (IsDelimited(type, '[', ']')) { return JSON_ARRAY; } + else if (IsDelimited(type, '{', '}')) + { + return JSON_OBJECT; + } else { return JSON_OBJECT; @@ -91,7 +96,7 @@ JsonTypeToStr(JsonType type) { switch (type) { - case JSON_OBJECT: + case JSON_OBJECT: return "JSON_OBJECT"; case JSON_ARRAY: return "JSON_ARRAY"; @@ -325,6 +330,7 @@ Main(Array * args) { char *fieldType; bool isArrType = false; + bool isObjType = false; JsonValue *requiredVal; JsonValue *ignoreVal; @@ -343,12 +349,18 @@ Main(Array * args) goto finish; } - if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']') + if (IsDelimited(fieldType, '[', ']')) { fieldType++; fieldType[strlen(fieldType) - 1] = '\0'; isArrType = true; } + else if (IsDelimited(fieldType, '{', '}')) + { + fieldType++; + fieldType[strlen(fieldType) - 1] = '\0'; + isObjType = true; + } if (!StrEquals(fieldType, "object") && !StrEquals(fieldType, "array") && @@ -374,6 +386,10 @@ Main(Array * args) { fieldType[strlen(fieldType)] = ']'; } + else if (isObjType) + { + fieldType[strlen(fieldType)] = '}'; + } requiredVal = HashMapGet(fieldObj, "required"); if (requiredVal && JsonValueType(requiredVal) != JSON_BOOLEAN) @@ -529,11 +545,13 @@ Main(Array * args) { cType = "double"; } - else if (StrEquals(fieldType, "object")) + else if (StrEquals(fieldType, "object") || + IsDelimited(fieldType, '{', '}')) { cType = "HashMap *"; } - else if (StrEquals(fieldType, "array") || (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')) + else if (StrEquals(fieldType, "array") || + IsDelimited(fieldType, '[', ']')) { cType = "Array *"; } @@ -542,7 +560,20 @@ Main(Array * args) cType = fieldType; } - StreamPrintf(headerFile, " %s %s;\n", cType, field); + if (IsDelimited(fieldType, '{', '}') || + IsDelimited(fieldType, '[', ']')) + { + char end = fieldType[strlen(fieldType) - 1]; + fieldType[strlen(fieldType) - 1] = '\0'; + + StreamPrintf(headerFile, " %s /* of %s */ %s;\n", cType, fieldType + 1, field); + + fieldType[strlen(fieldType)] = end; + } + else + { + StreamPrintf(headerFile, " %s %s;\n", cType, field); + } } StreamPrintf(headerFile, "} %s;\n\n", type); @@ -673,7 +704,134 @@ Main(Array * args) StreamPrintf(implFile, " out->%s = JsonValueAsObject(val);\n", key); StreamPrintf(implFile, " Free(val); /* Not JsonValueFree() because we want the inner value. */\n"); } - else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']') + else if (IsDelimited(fieldType, '{', '}')) + { + fieldType++; + fieldType[strlen(fieldType) - 1] = '\0'; + isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum"); + jsonType = isEnum ? JSON_STRING : TypeToJsonType(fieldType); + + StreamPrintf(implFile, " out->%s = HashMapCreate();\n", key); + StreamPrintf(implFile, " if (!out->%s)\n", key); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " *errp = \"Failed to allocate memory for %s.%s.\";\n", type, key); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " else\n"); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " HashMap *obj = JsonValueAsObject(val);\n"); + StreamPrintf(implFile, " char *objKey;\n"); + StreamPrintf(implFile, " JsonValue *v;\n"); + StreamPrintf(implFile, "\n"); + StreamPrintf(implFile, " while (HashMapIterate(obj, &objKey, (void **) &v))\n"); + StreamPrintf(implFile, " {\n"); + + if (StrEquals(fieldType, "integer") || + StrEquals(fieldType, "float") || + StrEquals(fieldType, "boolean")) + { + char *cType; + + if (StrEquals(fieldType, "integer")) + { + cType = "int64_t"; + } + else if (StrEquals(fieldType, "float")) + { + cType = "double"; + } + else if (StrEquals(fieldType, "boolean")) + { + cType = "bool"; + } + else + { + /* Should never happen */ + cType = NULL; + } + + *fieldType = toupper(*fieldType); + + StreamPrintf(implFile, " %s *ref;\n", cType); + StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType)); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " *errp = \"%s.%s{} contains an invalid value.\";\n", type, key); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " ref = Malloc(sizeof(%s));\n", cType); + StreamPrintf(implFile, " if (!ref)\n"); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " *errp = \"Unable to allocate memory for object value.\";\n"); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " *ref = JsonValueAs%s(v);\n", fieldType); + StreamPrintf(implFile, " HashMapSet(out->%s, objKey, ref);\n", key); + + *fieldType = tolower(*fieldType); + } + else if (StrEquals(fieldType, "string")) + { + StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType)); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " HashMapSet(out->%s, objKey, StrDuplicate(JsonValueAsString(v)));\n", key); + } + else if (StrEquals(fieldType, "object")) + { + StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType)); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " HashMapSet(out->%s, objKey, JsonDuplicate(JsonValueAsObject(v)));\n", key); + } + else + { + if (isEnum) + { + StreamPrintf(implFile, " int parseResult;\n"); + } + StreamPrintf(implFile, " %s *parsed;\n", fieldType); + StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType)); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " parsed = Malloc(sizeof(%s));\n", fieldType); + StreamPrintf(implFile, " if (!parsed)\n"); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " *errp = \"Unable to allocate memory for array value.\";\n"); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + if (isEnum) + { + StreamPrintf(implFile, " parseResult = %sFromStr(JsonValueAsString(v));\n", fieldType); + StreamPrintf(implFile, " *parsed = parseResult;\n"); + StreamPrintf(implFile, " if (parseResult == -1)\n"); + } + else + { + StreamPrintf(implFile, " if (!%sFromJson(JsonValueAsObject(v), parsed, errp))\n", fieldType); + } + StreamPrintf(implFile, " {\n"); + if (!isEnum) + { + StreamPrintf(implFile, " %sFree(parsed);\n", fieldType); + } + StreamPrintf(implFile, " Free(parsed);\n"); + StreamPrintf(implFile, " return false;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " HashMapSet(out->%s, objKey, parsed);\n", key); + } + + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " }\n"); + + fieldType[strlen(fieldType)] = '}'; + } + else if (IsDelimited(fieldType, '[', ']')) { fieldType++; fieldType[strlen(fieldType) - 1] = '\0'; @@ -915,7 +1073,73 @@ Main(Array * args) { StreamPrintf(implFile, " HashMapSet(json, \"%s\", JsonValueObject(JsonDuplicate(val->%s)));\n", Trim('_', key), key); } - else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']') + else if (IsDelimited(fieldType, '{', '}')) + { + int isPrimitive; + + fieldType++; + fieldType[strlen(fieldType) - 1] = '\0'; + isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum"); + isPrimitive = StrEquals(fieldType, "integer") || + StrEquals(fieldType, "boolean") || + StrEquals(fieldType, "float"); + + + StreamPrintf(implFile, " if (val->%s)\n", key); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " char *objKey;\n"); + StreamPrintf(implFile, " void *objVal;\n"); + StreamPrintf(implFile, " HashMap *jsonObj = HashMapCreate();\n"); + StreamPrintf(implFile, " if (!jsonObj)\n"); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " JsonFree(json);\n"); + StreamPrintf(implFile, " return NULL;\n"); + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " while (HashMapIterate(val->%s, &objKey, &objVal))\n", key); + StreamPrintf(implFile, " {\n"); + + if (StrEquals(fieldType, "string")) + { + StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, JsonValueString(objVal));\n", key); + } + else if (!isPrimitive) + { + StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, JsonValueObject(%sToJson(objVal)));\n", fieldType, key); + } + else + { + char *cType; + + if (StrEquals(fieldType, "integer")) + { + cType = "int64_t"; + } + else if (StrEquals(fieldType, "float")) + { + cType = "double"; + } + else if (StrEquals(fieldType, "boolean")) + { + cType = "bool"; + } + else + { + /* Should never happen */ + cType = NULL; + } + + *fieldType = toupper(*fieldType); + StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, (JsonValue%s(*((%s *) objVal))));\n", fieldType, cType, key); + *fieldType = tolower(*fieldType); + } + + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " HashMapSet(json, \"%s\", JsonValueObject(jsonObj));\n", Trim('_', key)); + StreamPrintf(implFile, " }\n"); + + fieldType[strlen(fieldType)] = '}'; + } + else if (IsDelimited(fieldType, '[', ']')) { int isPrimitive; @@ -1036,7 +1260,36 @@ Main(Array * args) { StreamPrintf(implFile, " Free(val->%s);\n", key); } - else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']') + else if (IsDelimited(fieldType, '{', '}')) + { + int isPrimitive; + + fieldType++; + fieldType[strlen(fieldType) - 1] = '\0'; + isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum"); + isPrimitive = StrEquals(fieldType, "boolean") || + StrEquals(fieldType, "float") || + StrEquals(fieldType, "integer") || + StrEquals(fieldType, "string"); + + StreamPrintf(implFile, " if (val->%s)\n", key); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " char *objKey;\n"); + StreamPrintf(implFile, " void *objVal;\n"); + StreamPrintf(implFile, " while (HashMapIterate(val->%s, &objKey, &objVal))\n", key); + StreamPrintf(implFile, " {\n"); + StreamPrintf(implFile, " %sFree(objVal);\n", (!isEnum && !isPrimitive) ? fieldType : "", key); + if (!isEnum && !isPrimitive) + { + StreamPrintf(implFile, " Free(objVal);\n", key); + } + StreamPrintf(implFile, " }\n"); + StreamPrintf(implFile, " HashMapFree(val->%s);\n", key); + StreamPrintf(implFile, " }\n"); + + fieldType[strlen(fieldType)] = '}'; + } + else if (IsDelimited(fieldType, '[', ']')) { int isPrimitive;