Enhance IMAP handling in RunACTIONS function and add example configuration for email extraction

Author: realdatadriven

.gitignore

@@ -40,4 +40,5 @@ logs.*.json
 *nyc*
 --public/
 examples/saas_model.md
+examples/downloads
 *.exe

examples/actions_email_ex.md

@@ -0,0 +1,56 @@
+# ACTIONS
+
+```yaml metadata
+name: FileOperations
+description: "Transfer and organize generated reports"
+path: examples
+active: true
+```
+
+## EMAILS
+```yaml
+name: emails
+description: Extract Emails
+type: IMAP
+params:
+    protocol: IMAP
+    host: imap.gmail.com
+    port: 993
+    username: "@SMTP_USERNAME"
+    password: "@SMTP_PASSWORD"
+    folder: INBOX
+    download_att: true
+    attachment_path: ./examples/downloads
+    search:
+        _from: supplier@example.com
+        subject: C7 Intro
+        since: 24h
+        _before: 24h
+    conn: "duckdb:"
+    sqls:
+        - ATTACH 'database/etl.db' AS DB (TYPE SQLITE)
+        - create_emails
+        - merge_into_emails
+        - DETACH DB
+active: true
+```
+
+```sql
+-- create_emails
+CREATE TABLE IF NOT EXISTS DB.emails  (
+    id           BIGINT,
+    subject      VARCHAR,
+    "from"       VARCHAR,
+    "to"         VARCHAR,
+    cc           VARCHAR,
+    bcc          VARCHAR,
+    date         TIMESTAMP,
+    body         TEXT,
+    attachments  VARCHAR
+);
+```
+
+```sql
+-- merge_into_emails
+INSERT INTO DB.emails BY NAME SELECT * FROM READ_JSON('<fname>')
+```

internal/etlx/imap.go

@@ -16,8 +16,12 @@ import (
 
 func returnAdresses(imapAdress []*imap.Address) string {
 	adress := ""
-	for _, adr := range imapAdress {
-		adress += ";" + adr.Address()
+	for i, adr := range imapAdress {
+		glue := ""
+		if i > 0 {
+			glue = ";"
+		}
+		adress += glue + adr.Address()
 	}
 	return adress
 }
@@ -48,17 +52,20 @@ func (etlx *ETLX) ReadEmails(cfg map[string]any, item map[string]any, dateRef []
 	// Connect IMAP
 	c, err := client.DialTLS(fmt.Sprintf("%s:%s", host, port), nil)
 	if err != nil {
+		fmt.Println("client.DialTLS Err:", err)
 		return nil, err
 	}
 	defer c.Logout()
 	// Login
 	err = c.Login(username, password)
 	if err != nil {
+		fmt.Println("c.Login Err:", err)
 		return nil, err
 	}
 	// Select mailbox
 	_, err = c.Select(folder, false)
 	if err != nil {
+		fmt.Println("c.Select(folder, false) Err:", err)
 		return nil, err
 	}
 	// Build search
@@ -69,7 +76,8 @@ func (etlx *ETLX) ReadEmails(cfg map[string]any, item map[string]any, dateRef []
 			case "from":
 				criteria.Header.Add("From", fmt.Sprint(value))
 			case "subject":
-				criteria.Header.Add("Subject", fmt.Sprint(value))
+				subj := etlx.SetQueryPlaceholders(fmt.Sprint(value), "", "", dateRef)
+				criteria.Header.Add("Subject", subj)
 			case "since":
 				d, err := time.ParseDuration(fmt.Sprint(value))
 				if err == nil {
@@ -85,8 +93,10 @@ func (etlx *ETLX) ReadEmails(cfg map[string]any, item map[string]any, dateRef []
 	}
 	ids, err := c.Search(criteria)
 	if err != nil {
+		fmt.Println("c.Search(criteria) Err:", err)
 		return nil, err
 	}
+	// fmt.Println("Fine Till Search Criteria OK")
 	results := []map[string]any{}
 	for _, id := range ids {
 		seq := new(imap.SeqSet)

internal/etlx/run_actions.go

@@ -468,7 +468,7 @@ func (etlx *ETLX) RunACTIONS(dateRef []time.Time, conf map[string]any, extraConf
 		case "imap", "IMAP":
 			_, okHost := params["host"].(string)
 			_, okPass := params["password"].(string)
-			_, okPort := params["port"].(string)
+			_, okPort := params["port"]
 			_, okUser := params["username"].(string)
 			conn, okConn := params["conn"].(string)
 			sqls, okSQLs := params["sqls"].([]any)
@@ -488,6 +488,7 @@ func (etlx *ETLX) RunACTIONS(dateRef []time.Time, conf map[string]any, extraConf
 				_log2["msg"] = fmt.Sprintf("%s -> %s -> %s: missing required params port", key, itemKey, _type)
 				valid = false
 			}
+			params["port"] = fmt.Sprintf("%v", params["port"])
 			if !okUser {
 				_log2["success"] = false
 				_log2["msg"] = fmt.Sprintf("%s -> %s -> %s: missing required params user", key, itemKey, _type)
@@ -542,6 +543,7 @@ func (etlx *ETLX) RunACTIONS(dateRef []time.Time, conf map[string]any, extraConf
 					}
 				}
 			}
+			// fmt.Println("IMAP:", _log2["msg"])
 		default:
 			_log2["success"] = false
 			_log2["msg"] = fmt.Sprintf("%s -> %s -> %s: Unsupported type", key, itemKey, _type)