Listing management

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}
+{"sqlitepath":"/db"}

index.js

@@ -1,6 +1,6 @@
 import fs from 'fs';
 import path from 'path';
-import { checkIfConfigIsAccessible, config, getProviders, refreshConfig } from './lib/utils.js';
+import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js';
 import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
 import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyPipeline from './lib/FredyPipeline.js';
@@ -12,46 +12,52 @@ import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
 import logger from './lib/services/logger.js';
 import { bus } from './lib/services/events/event-bus.js';
 import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
+import { getSettings } from './lib/services/storage/settingsStorage.js';
+import SqliteConnection from './lib/services/storage/SqliteConnection.js';
+
+//in the config, we store the path of the sqlite file, thus we must check if it is available
+const isConfigAccessible = await checkIfConfigIsAccessible();
+await SqliteConnection.init();
 
 // Load configuration before any other startup steps
 await refreshConfig();
 
-const isConfigAccessible = await checkIfConfigIsAccessible();
-
 if (!isConfigAccessible) {
   logger.error('Configuration exists, but is not accessible. Please check the file permission');
   process.exit(1);
 }
 
+// Run DB migrations once at startup and block until finished
+await runMigrations();
+
+const settings = await getSettings();
+
 // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
-const rawDir = config.sqlitepath || '/db';
+const rawDir = settings.sqlitepath || '/db';
 const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
 const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
 if (!fs.existsSync(absDir)) {
   fs.mkdirSync(absDir, { recursive: true });
 }
 
-// Run DB migrations once at startup and block until finished
-await runMigrations();
-
 // Load provider modules once at startup
 const providers = await getProviders();
 
 similarityCache.initSimilarityCache();
 similarityCache.startSimilarityCacheReloader();
 
 //assuming interval is always in minutes
-const INTERVAL = config.interval * 60 * 1000;
+const INTERVAL = settings.interval * 60 * 1000;
 
 // Initialize API only after migrations completed
 await import('./lib/api/api.js');
 
-if (config.demoMode) {
+if (settings.demoMode) {
   logger.info('Running in demo mode');
   cleanupDemoAtMidnight();
 }
 
-logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
 
 ensureAdminUserExists();
 ensureDemoUserExists();
@@ -65,10 +71,10 @@ bus.on('jobs:runAll', () => {
 });
 
 const execute = () => {
-  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-  if (!config.demoMode) {
+  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(settings, Date.now());
+  if (!settings.demoMode) {
     if (isDuringWorkingHoursOrNotSet) {
-      config.lastRun = Date.now();
+      settings.lastRun = Date.now();
       jobStorage
         .getJobs()
         .filter((job) => job.enabled)

lib/api/api.js

@@ -7,7 +7,6 @@ import { versionRouter } from './routes/versionRouter.js';
 import { loginRouter } from './routes/loginRoute.js';
 import { userRouter } from './routes/userRoute.js';
 import { jobRouter } from './routes/jobRouter.js';
-import { config } from '../utils.js';
 import bodyParser from 'body-parser';
 import restana from 'restana';
 import files from 'serve-static';
@@ -16,9 +15,11 @@ import { getDirName } from '../utils.js';
 import { demoRouter } from './routes/demoRouter.js';
 import logger from '../services/logger.js';
 import { listingsRouter } from './routes/listingsRouter.js';
+import { getSettings } from '../services/storage/settingsStorage.js';
+import { featureRouter } from './routes/featureRouter.js';
 const service = restana();
 const staticService = files(path.join(getDirName(), '../ui/public'));
-const PORT = config.port || 9998;
+const PORT = (await getSettings()).port || 9998;
 
 service.use(bodyParser.json());
 service.use(cookieSession());
@@ -39,6 +40,7 @@ service.use('/api/version', versionRouter);
 service.use('/api/jobs', jobRouter);
 service.use('/api/login', loginRouter);
 service.use('/api/listings', listingsRouter);
+service.use('/api/features', featureRouter);
 //this route is unsecured intentionally as it is being queried from the login page
 service.use('/api/demo', demoRouter);
 

lib/api/routes/demoRouter.js

@@ -1,10 +1,11 @@
 import restana from 'restana';
-import { config } from '../../utils.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const demoRouter = service.newRouter();
 
 demoRouter.get('/', async (req, res) => {
-  res.body = Object.assign({}, { demoMode: config.demoMode });
+  const settings = await getSettings();
+  res.body = Object.assign({}, { demoMode: settings.demoMode });
   res.send();
 });
 

lib/api/routes/featureRouter.js

@@ -0,0 +1,12 @@
+import restana from 'restana';
+import getFeatures from '../../features.js';
+const service = restana();
+const featureRouter = service.newRouter();
+
+featureRouter.get('/', async (req, res) => {
+  const features = getFeatures();
+  res.body = Object.assign({}, { features });
+  res.send();
+});
+
+export { featureRouter };

lib/api/routes/generalSettingsRoute.js

@@ -1,24 +1,30 @@
 import restana from 'restana';
-import { config, getDirName, readConfigFromStorage, refreshConfig } from '../../utils.js';
+import { getDirName } from '../../utils.js';
 import fs from 'fs';
 import { ensureDemoUserExists } from '../../services/storage/userStorage.js';
 import logger from '../../services/logger.js';
+import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
+
 generalSettingsRouter.get('/', async (req, res) => {
-  res.body = Object.assign({}, config);
+  res.body = Object.assign({}, await getSettings());
   res.send();
 });
 generalSettingsRouter.post('/', async (req, res) => {
-  const settings = req.body;
+  const { sqlitepath, ...appSettings } = req.body || {};
+  const localSettings = await getSettings();
+
+  if (localSettings.demoMode) {
+    res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+    return;
+  }
+
   try {
-    if (config.demoMode) {
-      res.send(new Error('In demo mode, it is not allowed to change these settings.'));
-      return;
+    if (typeof sqlitepath !== 'undefined') {
+      fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ sqlitepath }));
     }
-    const currentConfig = await readConfigFromStorage();
-    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ ...currentConfig, ...settings }));
-    await refreshConfig();
+    upsertSettings(appSettings);
     ensureDemoUserExists();
   } catch (err) {
     logger.error(err);

lib/api/routes/jobRouter.js

@@ -1,10 +1,10 @@
 import restana from 'restana';
 import * as jobStorage from '../../services/storage/jobStorage.js';
 import * as userStorage from '../../services/storage/userStorage.js';
-import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
 import logger from '../../services/logger.js';
 import { bus } from '../../services/events/event-bus.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 
 const service = restana();
 const jobRouter = service.newRouter();
@@ -44,9 +44,10 @@ jobRouter.get('/', async (req, res) => {
 });
 
 jobRouter.get('/processingTimes', async (req, res) => {
+  const settings = await getSettings();
   res.body = {
-    interval: config.interval,
-    lastRun: config.lastRun || null,
+    interval: settings.interval,
+    lastRun: settings.lastRun || null,
   };
   res.send();
 });

lib/api/routes/loginRoute.js

@@ -1,9 +1,9 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as hasher from '../../services/security/hash.js';
-import { config } from '../../utils.js';
 import { trackDemoAccessed } from '../../services/tracking/Tracker.js';
 import logger from '../../services/logger.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const loginRouter = service.newRouter();
 loginRouter.get('/user', async (req, res) => {
@@ -20,14 +20,15 @@ loginRouter.get('/user', async (req, res) => {
   res.send();
 });
 loginRouter.post('/', async (req, res) => {
+  const settings = await getSettings();
   const { username, password } = req.body;
   const user = userStorage.getUsers(true).find((user) => user.username === username);
   if (user == null) {
     res.send(401);
     return;
   }
   if (user.password === hasher.hash(password)) {
-    if (config.demoMode) {
+    if (settings.demoMode) {
       await trackDemoAccessed();
     }
 

lib/api/routes/userRoute.js

@@ -1,7 +1,7 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as jobStorage from '../../services/storage/jobStorage.js';
-import { config } from '../../utils.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const userRouter = service.newRouter();
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
@@ -23,7 +23,8 @@ userRouter.get('/:userId', async (req, res) => {
   res.send();
 });
 userRouter.delete('/', async (req, res) => {
-  if (config.demoMode) {
+  const settings = await getSettings();
+  if (settings.demoMode) {
     res.send(new Error('In demo mode, it is not allowed to remove user.'));
     return;
   }
@@ -44,7 +45,8 @@ userRouter.delete('/', async (req, res) => {
   res.send();
 });
 userRouter.post('/', async (req, res) => {
-  if (config.demoMode) {
+  const settings = await getSettings();
+  if (settings.demoMode) {
     res.send(new Error('In demo mode, it is not allowed to change or add user.'));
     return;
   }

lib/features.js

@@ -0,0 +1,9 @@
+const FEATURES = {
+  WATCHLIST_MANAGEMENT: false,
+};
+
+export default function getFeatures() {
+  return {
+    ...FEATURES,
+  };
+}

lib/services/crons/demoCleanup-cron.js

@@ -1,8 +1,8 @@
 import { removeJobsByUserId } from '../storage/jobStorage.js';
-import { config } from '../../utils.js';
 import { getUsers } from '../storage/userStorage.js';
 import logger from '../logger.js';
 import cron from 'node-cron';
+import { getSettings } from '../storage/settingsStorage.js';
 
 /**
  * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
@@ -11,12 +11,13 @@ export function cleanupDemoAtMidnight() {
   cron.schedule('0 0 * * *', cleanup);
 }
 
-function cleanup() {
-  if (config.demoMode) {
+async function cleanup() {
+  const settings = await getSettings();
+  if (settings.demoMode) {
     const demoUser = getUsers(false).find((user) => user.username === 'demo');
     if (demoUser == null) {
       logger.error('Demo user not found, cannot remove Jobs');
-      return;
+      return Promise.resolve();
     }
     removeJobsByUserId(demoUser.id);
   }

lib/services/crons/tracker-cron.js

@@ -1,10 +1,12 @@
 import cron from 'node-cron';
-import { config, inDevMode } from '../../utils.js';
+import { inDevMode } from '../../utils.js';
 import { trackMainEvent } from '../tracking/Tracker.js';
+import { getSettings } from '../storage/settingsStorage.js';
 
 async function runTask() {
+  const settings = await getSettings();
   //make sure to only send tracking events if the user gave us the green light and we are not in dev mode
-  if (config.analyticsEnabled && !inDevMode()) {
+  if (settings.analyticsEnabled && !inDevMode()) {
     await trackMainEvent();
   }
 }

lib/services/storage/SqliteConnection.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import Database from 'better-sqlite3';
 import logger from '../../services/logger.js';
-import { config } from '../../utils.js';
+import { readConfigFromStorage } from '../../utils.js';
 
 /**
  * SqliteConnection
@@ -25,16 +25,28 @@ import { config } from '../../utils.js';
 class SqliteConnection {
   static #db = null;
 
+  static #sqlLiteCfg = null;
+
+  static async init() {
+    if (this.#sqlLiteCfg == null) {
+      readConfigFromStorage().then((c) => {
+        this.#sqlLiteCfg = c.sqlitepath;
+      });
+    }
+  }
   /**
    * Returns a singleton instance of better-sqlite3 Database.
    * Respects env var SQLITE_DB_PATH and defaults to db/listings.db.
    */
   static getConnection() {
     if (this.#db) return this.#db;
 
+    if (this.#sqlLiteCfg == null) {
+      logger.warn('No sqlitepath configured. Using default db/listings.db');
+    }
+
     // Interpret config.sqlitepath as a directory relative to project root when it starts with '/'
-    const cfg = typeof config === 'object' && config ? config.sqlitepath : undefined;
-    const rawDir = cfg && cfg.length > 0 ? cfg : '/db';
+    const rawDir = this.#sqlLiteCfg && this.#sqlLiteCfg.length > 0 ? this.#sqlLiteCfg : '/db';
     const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
     const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
     const dbPath = path.join(absDir, 'listings.db');