4.0.8 (#31)

* Add support to remove records from encrypted kbs

* solve bug collating mz tables due to consistency error.

* Extend 'dump keys' command to mz tables. 

* Solve minor bug with version checking

Author: mscasso-scanoss

README.md

@@ -105,6 +105,9 @@ unlink list from DBNAME/TABLENAME key KEY
 
 dump DBNAME/TABLENAME hex N
     Dumps table contents with first N bytes in hex
+
+dump keys from DBNAME/TABLENAME [sector N]
+    Dumps a unique list of existing keys
 ```
 ## Other Uses
 

src/collate.c

@@ -23,6 +23,7 @@
 #include "mz.h"
 #include "collate.h"
 #include "logger.h"
+#include "decode.h"
 /**
   * @file collate.c
   * @date 19 Aug 2020 
@@ -524,11 +525,25 @@ bool key_in_delete_list(struct ldb_collate_data *collate, uint8_t *key, uint8_t
 
 				if (result)
 				{
-					char * aux = strndup((char*)data + (collate->del_tuples->keys_number - 1) * collate->del_tuples->key_ln, 
-										size - (collate->del_tuples->keys_number - 1) * collate->del_tuples->key_ln);
-
-					result = data_compare(collate->del_tuples->tuples[i]->data + char_to_skip, aux);
-					free(aux);
+					if (collate->in_table.definitions & LDB_TABLE_DEFINITION_ENCRYPTED)
+					{
+						unsigned char tuple_bin[MAX_CSV_LINE_LEN];
+						if(!decode && !ldb_decoder_lib_load())
+							return false;
+
+						int r_size = decode(DECODE_BASE64, NULL, NULL, collate->del_tuples->tuples[i]->data + char_to_skip, strlen(collate->del_tuples->tuples[i]->data) - char_to_skip, tuple_bin);
+						if (r_size > 0)
+							result = !memcmp(tuple_bin, data + (collate->del_tuples->keys_number - 1) * collate->del_tuples->key_ln, r_size);
+						else
+							result = false;
+					}
+					else
+					{
+						char * aux = strndup((char*)data + (collate->del_tuples->keys_number - 1) * collate->del_tuples->key_ln, 
+						size - (collate->del_tuples->keys_number - 1) * collate->del_tuples->key_ln);
+						result = data_compare(collate->del_tuples->tuples[i]->data + char_to_skip, aux);
+						free(aux);
+					}
 				}
 			}
 			/* Increment del_count and return true */

src/command.c

@@ -72,6 +72,7 @@ char *ldb_commands[] =
 	"dump {ascii} hex {ascii} sector {hex}",
 	"dump {ascii} hex {ascii}",
 	"dump keys from {ascii}",
+	"dump keys from {ascii} sector {hex}",
 	"cat {hex} from {ascii}"
 };
 int ldb_commands_count = sizeof(ldb_commands) / sizeof(ldb_commands[0]);
@@ -914,12 +915,29 @@ void ldb_command_dump_keys(char *command)
 {
 	/* Extract values from command */
 	char *dbtable = ldb_extract_word(4, command);
+	char * sector = ldb_extract_word(5, command);
+	int sectorn = -1;
+	if (*sector)
+	{
+		char *sector_n  = ldb_extract_word(6, command);
+		sectorn = (int) strtol(sector_n, NULL, 16);
+		free(sector_n);
+	}
+	free(sector);
 
 	if (ldb_valid_table(dbtable))
 	{
 			/* Assembly ldb table structure */
 			struct ldb_table ldbtable = ldb_read_cfg(dbtable);
-			ldb_dump_keys(ldbtable);
+			if ((ldbtable.definitions > 0 && ldbtable.definitions & LDB_TABLE_DEFINITION_MZ) ||
+				(!strcmp(ldbtable.table, "sources") || !strcmp(ldbtable.table, "notices")))
+			{
+				mz_list_keys(ldbtable, sectorn);
+			}
+			else
+			{
+				ldb_dump_keys(ldbtable, sectorn);
+			}
 	}
 
 	/* Free memory */

src/decode.c

@@ -0,0 +1,34 @@
+#include "decode.h"
+#include "logger.h"
+void * lib_handle = NULL;
+
+int (*decode) (int op, unsigned char *key, unsigned char *nonce,
+		        const char *buffer_in, int buffer_in_len, unsigned char *buffer_out) = NULL;
+
+bool ldb_decoder_lib_load(void)
+{
+	if (lib_handle != NULL)
+		return true;
+	/*set decode funtion pointer to NULL*/
+	decode = NULL;
+	lib_handle = dlopen("libscanoss_encoder.so", RTLD_NOW);
+	char * err;
+    if (lib_handle) 
+	{
+		log_info("Lib scanoss_encoder present\n");
+		decode = dlsym(lib_handle, "scanoss_decode");
+		if ((err = dlerror())) 
+		{
+			log_info("%s\n", err);
+			return false;
+		}
+		return true;
+     }
+	 
+	 return false;
+}
+
+void ldb_decoder_lib_close(void)
+{
+	dlclose(lib_handle);
+}

src/decode.h

@@ -0,0 +1,13 @@
+#ifndef __DECODE_H
+#define __DECODE_H
+#include <stdio.h>
+#include <dlfcn.h>
+#include <stdint.h>
+#include <stdbool.h>
+#define DECODE_BASE64 8
+extern int (*decode) (int op, unsigned char *key, unsigned char *nonce,
+		        const char *buffer_in, int buffer_in_len, unsigned char *buffer_out);
+extern void * lib_handle;
+bool ldb_decoder_lib_load(void);
+void ldb_decoder_lib_close(void);
+#endif

src/import.c

@@ -31,7 +31,6 @@
 #include <libgen.h>
 #include <dirent.h>
 #include <string.h>
-#include <dlfcn.h>
 #include <time.h>
 #include "import.h"
 #include "./ldb.h"
@@ -44,48 +43,16 @@
 #include "collate.h"
 #include "logger.h"
 #include "ldb_error.h"
-
-#define DECODE_BASE64 8
-#define MAX_CSV_LINE_LEN 1024
+#include "decode.h"
 #define REC_SIZE_LEN 2
 
 pthread_mutex_t lock;
 
 int max_threads = 0;
 pthread_t * threads_list;
 static long IGNORED_WFP_LN = sizeof(IGNORED_WFP);
-
-int (*decode) (int op, unsigned char *key, unsigned char *nonce,
-		        const char *buffer_in, int buffer_in_len, unsigned char *buffer_out) = NULL;
-
 double progress_timer = 0;
 
-
-void * lib_handle = NULL;
-bool ldb_import_decoder_lib_load(void)
-{
-	if (lib_handle != NULL)
-		return true;
-	/*set decode funtion pointer to NULL*/
-	decode = NULL;
-	lib_handle = dlopen("libscanoss_encoder.so", RTLD_NOW);
-	char * err;
-    if (lib_handle) 
-	{
-		log_info("Lib scanoss_encoder present\n");
-		decode = dlsym(lib_handle, "scanoss_decode");
-		if ((err = dlerror())) 
-		{
-			log_info("%s\n", err);
-			return false;
-		}
-		return true;
-     }
-	 
-	 return false;
-}
-
-
 /**
  * @brief Sort a csv file invokinkg a new process and executing a sort command
  *
@@ -521,6 +488,20 @@ int ldb_import_mz(ldb_importation_config_t * job)
 	if (!ldb_table_exists(job->dbname, job->table))
 		ldb_create_table_new(job->dbname, job->table, 16, 0, job->opt.params.keys_number, LDB_TABLE_DEFINITION_MZ |
 						(job->opt.params.binary_mode ? LDB_TABLE_DEFINITION_ENCRYPTED : LDB_TABLE_DEFINITION_STANDARD));
+	else
+	{
+			/* Create table structure for bulk import (32-bit key) */
+		char db_table[LDB_MAX_NAME];
+		snprintf(db_table,LDB_MAX_NAME-1, "%s/%s", job->dbname, job->table);
+		struct ldb_table oss_bulk = ldb_read_cfg(db_table);
+		if (oss_bulk.keys < 1 || oss_bulk.definitions == LDB_TABLE_DEFINITION_UNDEFINED)
+		{
+			oss_bulk.keys = job->opt.params.keys_number;
+			ldb_write_cfg(oss_bulk.db, oss_bulk.table, oss_bulk.key_ln, oss_bulk.rec_ln, oss_bulk.keys, 
+						job->opt.params.binary_mode ? LDB_TABLE_DEFINITION_MZ | LDB_TABLE_DEFINITION_ENCRYPTED :  LDB_TABLE_DEFINITION_MZ | LDB_TABLE_DEFINITION_STANDARD);
+			log_info("Table %s config file was updated\n", oss_bulk.table);
+		}
+	}
 
 	return (ldb_bin_join(job->csv_path, dest_path, job->opt.params.overwrite, false, job->opt.params.delete_after_import));
 }
@@ -1491,7 +1472,7 @@ int ldb_import(ldb_importation_config_t * job)
 	{
 		pthread_mutex_lock(&lock);
 		if(!decode)
-			ldb_import_decoder_lib_load();
+			ldb_decoder_lib_load();
 		pthread_mutex_unlock(&lock);
 	}
 
@@ -1820,7 +1801,7 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 	import_params_t user_opt = {.params = LDB_IMPORTATION_CONFIG_UNDEFINED};
 	import_params_t global_opt = {.params = LDB_IMPORTATION_CONFIG_UNDEFINED};
 	ldb_importation_config_t job = {.opt.params = LDB_IMPORTATION_CONFIG_DEFAULT, .csv_path = "\0", .path = "\0"};
-	
+
 	char *table = strrchr(dbtable, '/');
 	if (table)
 		strncpy(job.dbname, dbtable, table - dbtable);
@@ -1863,18 +1844,19 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 		strcpy(job.path, path);
 			//check if the version file is present and update the kb version.
 		bool version_present = version_import(&job);
+
+		recurse_directory(&jobs, path, NULL);
+		opt_add(jobs.user_opt, jobs.global_opt);
+		opt_add(jobs.global_opt, jobs.user_opt);
+
 		//abort the job if VERSION_VALIDATION is active and the json file is not present
 		if (job.opt.params.version_validation && !version_present)
 		{
-			logger_basic("Failed to validate version.json, check if it is present in %s and it has the correct format\n", job.path);
-			logger_basic("You can avoid this error by setting \"VALIDATE_VERSION = 0\" in the import configuration options\n", job.path);
-
+			fprintf(stderr, "Failed to validate version.json, check if it is present in %s and it has the correct format\n", job.path);
+			fprintf(stderr, "You can avoid this error by setting \"VALIDATE_VERSION = 0\" in the import configuration options\n");
 			exit(EXIT_FAILURE);
 		}
 
-		recurse_directory(&jobs, path, NULL);
-		opt_add(jobs.user_opt, jobs.global_opt);
-		opt_add(jobs.global_opt, jobs.user_opt);
 		print_jobs(&jobs);
 		max_threads = jobs.user_opt->params.threads;
 		fprintf(stderr, "Max threads set to: %d\n", max_threads);
@@ -1938,18 +1920,19 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 			strcpy(job.path,path);
 		}
 
+		load_import_config(&job, &global_opt);
+		opt_add(&user_opt, &global_opt);
+		opt_add(&user_opt, &job.opt);
+
 		bool version_present = version_import(&job);
 		//abort the job if VERSION_VALIDATION is active and the json file is not present
 		if (job.opt.params.version_validation && !version_present)
 		{
-			logger_basic("Failed to validate version.json, check if it is present in %s and it has the correct format\n", job.path);
+			fprintf(stderr, "Failed to validate version.json, check if it is present in %s and it has the correct format\n", job.path);
+			fprintf(stderr, "You can avoid this error by setting \"VALIDATE_VERSION = 0\" in the import configuration options\n");
 			exit(EXIT_FAILURE);
 		}
 
-		load_import_config(&job, &global_opt);
-		opt_add(&user_opt, &global_opt);
-		opt_add(&user_opt, &job.opt);
-
 		max_threads = job.opt.params.threads;
 		fprintf(stderr, "Max threads set to: %d\n", max_threads);
 		pthread_mutex_init(&lock, NULL);

src/import.h

@@ -2,6 +2,7 @@
 #define __IMPORT_H
 #include <stdbool.h>
 #include "ldb.h"
+ 
 #define IMPORT_PARAMS_NUMBER 15
 typedef union import_params {
 	struct __attribute__((__packed__)) params

src/keys.c

@@ -56,12 +56,9 @@ bool ldb_dump_keys_handler(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t
 	/* Save into last key */
 	memcpy(table->last_key, table->current_key, table->key_ln);
 
-	/* Output key to stdout */
-	for (int i = 0; i < table->key_ln; i++) 
-	{
-	    fwrite(table->current_key + i, 1, 1, stdout);
-	}
-
+	char hex[MD5_LEN * 2 + 1];
+	ldb_bin_to_hex(table->current_key, MD5_LEN, hex);
+	printf("%s\n", hex);
 	return false;
 }
 
@@ -70,10 +67,10 @@ bool ldb_dump_keys_handler(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t
  * 
  * @param table input table
  */
-void ldb_dump_keys(struct ldb_table table)
+void ldb_dump_keys(struct ldb_table table, int s)
 {
 	/* Read each DB sector */
-	uint8_t k0 = 0;
+	uint8_t k0 = s >= 0 ? s : 0;
 	setlocale(LC_NUMERIC, "");
 
 	table.current_key = calloc(table.key_ln, 1);
@@ -93,14 +90,14 @@ void ldb_dump_keys(struct ldb_table table)
 						k[1] = k1;
 						k[2] = k2;
 						k[3] = k3;
-						/* If there is a pointer, read the list */
-						if (ldb_map_pointer_pos(k))
-						{
-							/* Process records */
-							ldb_fetch_recordset(sector, table, k, true, ldb_dump_keys_handler, &table);
-						}
+						
+						/* Process records */
+						ldb_fetch_recordset(sector, table, k, true, ldb_dump_keys_handler, &table);
+						
 					}
 			free(sector);
+			if (s >=0)
+				break;
 		}
 	} while (k0++ < 255);
 

src/ldb.h

@@ -27,7 +27,7 @@
 #include "./ldb/types.h"
 #include "./ldb/mz.h"
 
-#define LDB_VERSION "4.0.7"
+#define LDB_VERSION "4.0.8"
 
 #define LDB_TABLE_DEFINITION_UNDEFINED -1
 #define LDB_TABLE_DEFINITION_STANDARD 0
@@ -90,7 +90,7 @@ bool ldb_hexprint_width(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *d
 void ldb_sector_update(struct ldb_table table, uint8_t *key);
 void ldb_sector_erase(struct ldb_table table, uint8_t *key);
 void ldb_dump(struct ldb_table table, int hex_bytes, int sector);
-void ldb_dump_keys(struct ldb_table table);
+void ldb_dump_keys(struct ldb_table table, int s);
 
 char * ldb_file_extension(char * path);
 bool ldb_create_dir(char *path);

src/ldb/definitions.h

@@ -16,6 +16,7 @@
 #define MD5_LEN 16
 #define MD5_LEN_HEX 32
 #define BUFFER_SIZE 1048576
+#define MAX_CSV_LINE_LEN 1024
 
 extern char ldb_root[];
 extern char ldb_lock_path[];