3.1.6 (#22)

* update md5 calculation using gcrypt

* update github actions

Author: mscasso-scanoss

.github/workflows/build.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Install third party dependencies
-        run: sudo apt install build-essential libssl-dev zlib1g-dev libsqlite3-dev libz-dev curl gem ruby unzip p7zip-full unrar-free
+        run: sudo apt install build-essential libgcrypt-dev zlib1g-dev libsqlite3-dev libz-dev
 
       - uses: actions/checkout@v3
 

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
 
       - name: Install third party dependencies
-        run: sudo apt install build-essential libssl-dev zlib1g-dev
+        run: sudo apt install build-essential libgcrypt-dev zlib1g-dev
 
       - uses: actions/checkout@v3
         with:

Makefile

@@ -3,7 +3,7 @@ CC=gcc
 endif
 CCFLAGS?=-O -g -Wall -std=gnu99
 LIBFLAGS=$(CCFLAGS) -fPIC -c
-LIBS=-lm -lpthread -lz 
+LIBS=-lm -lpthread -lz -lgcrypt
 
 VERSION=$(shell ./version.sh)
 
@@ -18,13 +18,13 @@ help: ## This help
 all: clean lib shell ## Clean dev data and build the ldb's library and shell binaries
 
 lib: src/ldb.c src/mz.c src/ldb.h ## Build only the ldb library
-	@$(CC) $(LIBFLAGS) -D_LARGEFILE64_SOURCE src/ldb.c src/mz.c $(LIBS)
-	@$(CC) -shared -Wl,-soname,libldb.so -o libldb.so ldb.o mz.o $(LIBS)
+	@$(CC) $(LIBFLAGS) -D_LARGEFILE64_SOURCE src/ldb.c src/mz.c src/md5.c $(LIBS)
+	@$(CC) -shared -Wl,-soname,libldb.so -o libldb.so ldb.o mz.o md5.o $(LIBS)
 	@echo Library is built
 
 shell: src/shell.c src/command.c ## Build only the shell binary
-	@$(CC) $(CCFLAGS) -D_LARGEFILE64_SOURCE -c src/shell.c src/mz.c $(LIBS)
-	@$(CC) $(CCFLAGS) -o ldb ldb.o shell.o -lcrypto $(LIBS)
+	@$(CC) $(CCFLAGS) -D_LARGEFILE64_SOURCE -c src/shell.c src/mz.c src/md5.c $(LIBS)
+	@$(CC) $(CCFLAGS) -o ldb ldb.o shell.o md5.o $(LIBS)
 	@echo Shell is built
 
 static: src/ldb.c src/ldb.h src/shell.c ## Static build of the shell binary

src/ldb.h

@@ -36,7 +36,7 @@
 #include <unistd.h>
 #include <openssl/md5.h>
 
-#define LDB_VERSION "3.1.5"
+#define LDB_VERSION "3.2.0"
 #define LDB_MAX_PATH 1024
 #define LDB_MAX_NAME 64
 #define LDB_MAX_RECORDS 500000 // Max number of records per list
@@ -277,8 +277,7 @@ void mz_id_fill(char *md5, uint8_t *mz_id);
 void mz_parse(struct mz_job *job, bool (*mz_parse_handler) ());
 void file_write(char *filename, uint8_t *src, uint64_t src_ln);
 void mz_id_fill(char *md5, uint8_t *mz_id);
-void mz_deflate(struct mz_job *job);
-void mz_corrupted(void);
+void mz_corrupted();
 void mz_add(char *mined_path, uint8_t *md5, char *src, int src_ln, bool check, uint8_t *zsrc, struct mz_cache_item *mz_cache);
 bool mz_check(char *path);
 void mz_flush(char *mined_path, struct mz_cache_item *mz_cache);
@@ -288,4 +287,9 @@ void mz_cat(struct mz_job *job, char *key);
 uint8_t *file_md5 (char *path);
 void calc_md5(char *data, int size, uint8_t *out);
 
-//normalized_license *load_licenses();
+void md5_string(const unsigned char *input, int len, unsigned char output[16]);
+uint8_t * md5_file(char *path);
+
+#define MD5(a, b, c)  md5_string(a, b, c)
+#define MZ_DEFLATE(job) mz_deflate(job)
+

src/md5.c

@@ -0,0 +1,53 @@
+#include "ldb.h"
+#include <gcrypt.h>
+
+/* Adapter function for compatibility with openssl*/
+void md5_string(const unsigned char *input, int len, unsigned char output[16]) 
+{
+    gcry_md_hd_t h;
+    gcry_md_open(&h, GCRY_MD_MD5, GCRY_MD_FLAG_SECURE);  // initialize the hash context
+    gcry_md_write(h, input, len);  // hash the input data
+    const unsigned char *hash_result = gcry_md_read(h, GCRY_MD_MD5);  // get the result
+
+    // Copy the hash result to the output array
+    memcpy(output, hash_result, 16);
+
+    // Free memory for the hash result
+    gcry_md_close(h);
+}
+
+/**
+ * @brief Returns the hexadecimal md5 sum for "path"
+ * 
+ * @param path string path
+ * @return pointer to file md5 array
+ */
+uint8_t *file_md5(char *path) 
+{
+    uint8_t *c = calloc(1, gcry_md_get_algo_dlen(GCRY_MD_MD5));  // Allocate memory for MD5 hash
+    FILE *fp = fopen(path, "rb");
+
+    if (!fp) {
+        fprintf(stderr, "Unable to open file for reading.\n");
+        return NULL;
+    }
+
+    gcry_md_hd_t md5_hd;
+    gcry_md_open(&md5_hd, GCRY_MD_MD5, GCRY_MD_FLAG_SECURE);
+
+    uint8_t *buffer = malloc(BUFFER_SIZE);
+    size_t bytes;
+
+    while ((bytes = fread(buffer, 1, BUFFER_SIZE, fp)) > 0) {
+        gcry_md_write(md5_hd, buffer, bytes);
+    }
+
+    fclose(fp);
+    free(buffer);
+
+    const uint8_t *digest = gcry_md_read(md5_hd, GCRY_MD_MD5);
+    memcpy(c, digest, gcry_md_get_algo_dlen(GCRY_MD_MD5));
+
+    gcry_md_close(md5_hd);
+    return c;
+}

src/mz.c

@@ -30,7 +30,6 @@
   * @see https://github.com/scanoss/ldb/blob/master/src/lock.c
   */
 
-#include <openssl/md5.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <stdbool.h>
@@ -41,48 +40,6 @@
 #include <zlib.h>
 #include "ldb.h"
 
-/**
- * @brief Returns the hexadecimal md5 sum for "path"
- * 
- * @param path string path
- * @return pointer to file md5 array
- */
-uint8_t *file_md5 (char *path)
-{
-	uint8_t *c = calloc(16,1);
-	FILE *fp = fopen(path, "rb");
-	MD5_CTX mdContext;
-	uint32_t bytes;
-
-	if (fp != NULL)
-	{
-		uint8_t *buffer = malloc(BUFFER_SIZE);
-		MD5_Init (&mdContext);
-
-		while ((bytes = fread(buffer, 1, BUFFER_SIZE, fp)) != 0)
-			MD5_Update(&mdContext, buffer, bytes);
-
-		MD5_Final(c, &mdContext);
-		fclose(fp);
-		free(buffer);
-	}
-	return c;
-}
-
-/**
- * @brief Calculates the MD5 hash for data
- * 
- * @param data input data
- * @param size data size
- * @param out[out] pointer to MD5 array
- */
-void calc_md5(char *data, int size, uint8_t *out)
-{
-	MD5_CTX mdContext;
-	MD5_Init (&mdContext);
-	MD5_Update(&mdContext, data, size);
-	MD5_Final(out, &mdContext);
-}
 
 /**
  * @brief compare two MZ keys
@@ -169,7 +126,7 @@ bool mz_list_handler(struct mz_job *job)
 
 	/* Calculate resulting data MD5 */
 	uint8_t actual_md5[MD5_LEN];
-	calc_md5(job->data, job->data_ln, actual_md5);
+	md5_string((unsigned char*) job->data, job->data_ln, actual_md5);
 
 	/* Compare data checksum to validate */
 	char actual[MD5_LEN * 2 + 1] = "\0";
@@ -322,7 +279,7 @@ bool mz_extract_handler(struct mz_job *job)
 
 	/* Calculate resulting data MD5 */
 	uint8_t actual_md5[MD5_LEN];
-	calc_md5(job->data, job->data_ln, actual_md5);
+	md5_string((unsigned char*) job->data, job->data_ln, actual_md5);
 
 	/* Compare data checksum to validate */
 	char actual[MD5_LEN * 2 + 1] = "\0";
@@ -644,7 +601,7 @@ void mz_add(char *mined_path, uint8_t *md5, char *src, int src_ln, bool check, u
 	/* We save the first bytes of zsrc to accomodate the MZ header */
 	compress(zsrc + MZ_HEAD, &zsrc_ln, (uint8_t *) src, src_ln + 1);
 	uint32_t zln = zsrc_ln;
-
+	
 	/* Only the last 14 bytes of the MD5 go to the mz record (first two bytes are the file name) */
 	memcpy(zsrc, md5 + 2, MZ_MD5); 
 
@@ -653,7 +610,6 @@ void mz_add(char *mined_path, uint8_t *md5, char *src, int src_ln, bool check, u
 
 	int mzid = uint16(md5);
 	int mzlen = zsrc_ln + MZ_HEAD;
-
 	/* If it won't fit in the cache, write it directly */
 	if (mzlen > MZ_CACHE_SIZE)
 	{
@@ -767,18 +723,68 @@ void mz_corrupted()
 	exit(EXIT_FAILURE);
 }
 
+
 /**
  * @brief Decompress a MZ job
  * 
  * @param job MZ job
  */
+#define CHUNK_SIZE 1024
+
+int uncompress_by_chunks(uint8_t **data, uint8_t *zdata, size_t zdata_len) {
+    int ret;
+    z_stream strm;
+    unsigned char out[CHUNK_SIZE];
+    size_t data_size = 0;  // Current size of decompressed data
+
+    // Initialize the z_stream structure
+    memset(&strm, 0, sizeof(strm));
+    ret = inflateInit(&strm);
+    if (ret != Z_OK) {
+        fprintf(stderr, "inflateInit failed with error %d\n", ret);
+        exit(EXIT_FAILURE);
+    }
+	*data = malloc(CHUNK_SIZE);
+    // Process the compressed data
+    strm.avail_in = zdata_len;  // Size of the compressed data
+    strm.next_in = zdata;
+
+    do {
+        strm.avail_out = CHUNK_SIZE;
+        strm.next_out = out;
+
+        ret = inflate(&strm, Z_NO_FLUSH);
+        if (ret == Z_STREAM_ERROR) {
+            fprintf(stderr, "inflate failed with error Z_STREAM_ERROR\n");
+            inflateEnd(&strm);
+			mz_corrupted();
+        }
+
+        unsigned have = CHUNK_SIZE - strm.avail_out;
+
+        // Realloc to increase the size of data
+        *data = realloc(*data, data_size + have);
+        if (*data == NULL) 
+		{
+            fprintf(stderr, "Error reallocating memory to store decompressed data");
+            inflateEnd(&strm);
+            exit(EXIT_FAILURE);
+        }
+
+        // Copy the decompressed data to the end of data
+        memcpy(*data + data_size, out, have);
+        data_size += have;
+    } while (ret != Z_STREAM_END);
+
+    // Free resources
+    inflateEnd(&strm);
+	return data_size;
+}
+
 void mz_deflate(struct mz_job *job)
 {
 	/* Decompress data */
-	job->data_ln = MZ_MAX_FILE;
-	if (Z_OK != uncompress((uint8_t *)job->data, &job->data_ln, job->zdata, job->zdata_ln))
-	{
-		mz_corrupted();
-	}
+	job->data_ln = uncompress_by_chunks((uint8_t **) &job->data, job->zdata, job->zdata_ln);
 	job->data_ln--;
 }
+