Change in config default values, correction in README, solve bug with version validation, add version validation test. (#27)

Author: mscasso-scanoss

README.md

@@ -60,14 +60,14 @@ bulk insert DBNAME/TABLENAME from PATH with (CONFIG)
         KEYS: Number of binary keys in the CSV file.
         MZ: MZ file indicator.
         WFP: WFP file indicator.
-        OVERWRITE: Overwrite the destination table.
-        SORT: Sort the tuples during the import process.
+        OVERWRITE: Overwrite the destination table. (Default: 0).
+        SORT: Sort the tuples during the import process. (Default: 1).
         FIELDS: Number of CSV fields.
-        VALIDATE_FIELDS: Check field quantity during importation.
-        VALIDATE_VERSION: Validate version.json.
-        VERBOSE: Enable verbose mode.
+        VALIDATE_FIELDS: Check field quantity during importation. (Default: 1).
+        VALIDATE_VERSION: Validate version.json.(Default: 1).
+        VERBOSE: Enable verbose mode. (Default: 0).
         COLLATE: Perform collation after import, removing data larger than MAX_RECORD bytes.
-        MAX_RECORD: Maximum record size in bytes (default: 1024).
+        MAX_RECORD: Maximum record size in bytes (Default: 2048).
         MAX_RAM_PERCENT: limit the system RAM usage during collate process. Default value: 50.
         TMP_PATH: Path to the folder used for temporary files (default: /tmp).
 

src/import.c

@@ -1036,7 +1036,7 @@ bool version_import(ldb_importation_config_t *job)
 		test_len -= (strlen(daily_date_i) + strlen("\"daily\":"));
 	if (monthly_date_i)
 		test_len -= (strlen(monthly_date_i) + strlen("\"monthly\":"));
-	//exit if cannot find daily or monthly or there are am excess of characteres in the json
+	//exit if cannot find daily or monthly or there are an excess of characteres in the json
 	if ((!daily_date_i && !monthly_date_i) || test_len > 10)
 	{
 		logger_basic("Failed to process version file: %s\n", vf_import);
@@ -1072,7 +1072,7 @@ bool version_import(ldb_importation_config_t *job)
 		return false;
 	}
 
-	fprintf(f, JSON_CONTENT, monthly_date_i == NULL ? "N/A" : monthly_date_i ,daily_date_i == NULL ? "N/A" :  daily_date_i);
+	fprintf(f, JSON_CONTENT, monthly_date_i == NULL ? "N/A" : monthly_date_i, daily_date_i == NULL ? "N/A" :  daily_date_i);
 	fclose(f);
 	free(daily_date_i);
 	free(daily_date_o);
@@ -1176,7 +1176,7 @@ bool ldb_importation_config_parse(ldb_importation_config_t * config, char * line
 
 bool ldb_create_db_config_default(char * dbname)
 {
-	char config[] = "GLOBAL: (VALIDATE_FIELDS=1, VALIDATE_VERSION=0, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/tmp)\n"
+	char config[] = "GLOBAL: (VALIDATE_FIELDS=1, VALIDATE_VERSION=1, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/tmp)\n"
 					"sources: (MZ=1, KEYS=1)\n"
 					"notices: (MZ=1, KEYS=1)\n"
 					"attribution: (KEYS=1, FIELDS=2)\n"
@@ -1413,13 +1413,6 @@ int ldb_import(ldb_importation_config_t * job)
 	int result = -1;
 	ldb_importation_config_t config = *job;
 
-	logger_set_level(config.opt.params.verbose);
-	if (config.opt.params.version_validation && !version_import(&config))
-	{
-		logger_basic("Failed to validate version.json, check if it is present in %s and it has the correct format\n", config.path);
-		exit(EXIT_FAILURE);
-	}
-	
 	if (strstr(config.csv_path, ".mz"))
 		config.opt.params.is_mz_table = true;
 	else
@@ -1807,7 +1800,7 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 
 	if (!ldb_database_exists(job.dbname))
 		ldb_create_database(job.dbname);
-
+	
 	struct ldb_importation_jobs_s jobs = {.job = NULL, .number = 0, .common_opt = job.opt};
 	strcpy(jobs.dbname, job.dbname);
 	jobs.unsorted_index = 0;
@@ -1828,6 +1821,15 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 	if (!table && ldb_dir_exists(path))
 	{
 		strcpy(job.path, path);
+			//check if the version file is present and update the kb version.
+		bool version_present = version_import(&job);
+		//abort the job if VERSION_VALIDATION is active and the json file is not present
+		if (job.opt.params.version_validation && !version_present)
+		{
+			logger_basic("Failed to validate version.json, check if it is present in %s and it has the correct format\n", job.path);
+			exit(EXIT_FAILURE);
+		}
+
 		recurse_directory(&jobs, path, NULL);		
 		print_jobs(&jobs);
 		/* Process jobs*/
@@ -1882,6 +1884,13 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 		{
 			strcpy(job.path,path);
 		}
+		bool version_present = version_import(&job);
+		//abort the job if VERSION_VALIDATION is active and the json file is not present
+		if (job.opt.params.version_validation && !version_present)
+		{
+			logger_basic("Failed to validate version.json, check if it is present in %s and it has the correct format\n", job.path);
+			exit(EXIT_FAILURE);
+		}
 		load_import_config(&job);
 		process_sectors(&job, threads_list);
 	}

src/ldb.h

@@ -27,7 +27,7 @@
 #include "./ldb/types.h"
 #include "./ldb/mz.h"
 
-#define LDB_VERSION "4.0.3"
+#define LDB_VERSION "4.0.4"
 
 #define LDB_TABLE_DEFINITION_UNDEFINED -1
 #define LDB_TABLE_DEFINITION_STANDARD 0

test/source/mined2/quality.csv

@@ -0,0 +1 @@
+b377b77401a8bfbef9ea316787ebda99,0,4

test/source/mined2/version.json

@@ -0,0 +1 @@
+{"daily":"24.10.23"}

test/source/mined3/quality.csv

@@ -0,0 +1 @@
+b377b77401a8bfbef9ea316787ebda99,0,4

test/source/mined3/version.json

@@ -0,0 +1 @@
+{"daily":"25.10.23"}

test/test_kb

@@ -56,6 +56,15 @@ test_8_unicode_chars() {
     assert_equals "b337487d67c0307bb9d009e2b5e248c8,e7bc2c903379c1eda3ec7a12e88b933d,系列1-MyBatis源码解析/第10节课/mybatis-3-master/src/test/java/org/apache/ibatis/submitted/language/VelocitySqlSourceBuilder.java" $result
 }
 
+test_9_version_validation() {
+    ../ldb -q -u source/mined2 -n test_kb -q
+    result=$(cat /var/lib/ldb/test_kb/version.json)
+    assert_equals "{\"monthly\":\"23.10\", \"daily\":\"24.10.23\"}" "$result"
+    echo " bulk insert test_kb from source/mined3 with (FILE_DEL=0,VALIDATE_VERSION=0)" | ../ldb
+    result=$(cat /var/lib/ldb/test_kb/version.json)
+    assert_equals "{\"monthly\":\"23.10\", \"daily\":\"25.10.23\"}" "$result"
+}
+
 setup_suite () {
     ../ldb -u source/mined -n test_kb
 }