correct merge finction

Author: djadar

main.R

@@ -2,8 +2,9 @@ source("/home/amukam/thss/experiments/uniform-sample/dkl.r")
 
 experiment =  "KL1"
 "--------------------------"
-times <- 2
-for (i in 1:times){
+d1 <- divergence("data/resultJul95trunk")
+times <- 5
+for (i in 3:times){
     output = paste("data/output", i, sep="")
     dKL(c(i, output, experiment))
 }

main.go

@@ -15,12 +15,13 @@ func main() {
 	input_path = *path
 	switch *number {
 	case 0:
-		//test0()
-		check_cms()
+		test0()
+		//check_cms()
 	case 1:
 		test_omniscient()
 	case 2:
-		test_Knowledge_free()
+		//test_Knowledge_free()
+		test_Knowledge_free_with_small_set()
 	default:
 		panic("Wrong number")
 	}

network/countminsketch.go

@@ -48,6 +48,17 @@ func (cms *CMS) getMinMatrix() (min uint64) {
 	return
 }
 
+func getMinFreq() (min uint64) {
+	min = 0
+	for i := range in_memory {
+		freq := PeerCMS.EstimateString(i)
+		if freq < min || min == 0 {
+			min = freq
+		}
+	}
+	return
+}
+
 func (cms *CMS) Knowledge_free(peer string) (output_choice string) {
 	var cmsRand = rand.New(
 		rand.NewSource(time.Now().UnixNano())) /* RNG generator */
@@ -59,25 +70,34 @@ func (cms *CMS) Knowledge_free(peer string) (output_choice string) {
 		}
 
 	} else {
-		freq := PeerCMS.EstimateString(peer)
-		if freq == 0 {
-			log.Panicf("Element %s has not been registered yet !!!\n", peer)
+
+		var prob float64
+		if !in_memory[peer] {
+			prob = 1.0
+		} else {
+			//min := cms.getMinMatrix()
+			min := getMinFreq()
+			freq := PeerCMS.EstimateString(peer)
+			if freq == 0 {
+				log.Panicf("Element %s has not been registered yet !!!\n", peer)
+			}
+			prob = float64(min) / float64(freq) // aj <= 1
+			//fmt.Println("min", min, "prob", prob)
 		}
-		min := cms.getMinMatrix()
 
-		prob := float64(min) / float64(freq) // aj <= 1
-		choice := cmsRand.Float64()          // random choice in [0.0, 1.0[
+		choice := cmsRand.Float64() // random choice in [0.0, 1.0[
+		//fmt.Println("choice", choice)
+		if choice < prob && !in_memory[peer] { // try to add only not known elements
 
-		if choice < prob {
 			sample_choice_index := cmsRand.Intn(C) //uniform random choice
-			if !in_memory[peer] {
-				k := Sample_memory[sample_choice_index]
-				delete(in_memory, k)
-				Sample_memory[sample_choice_index] = peer // add j
-				in_memory[peer] = true
-			}
+			k := Sample_memory[sample_choice_index]
+			delete(in_memory, k)
+			Sample_memory[sample_choice_index] = peer // add j
+			in_memory[peer] = true
+			//fmt.Println("change", k, peer)
 		}
 	}
+	//fmt.Println("Memory", Sample_memory)
 	output_choice_index := cmsRand.Intn(len(Sample_memory)) //uniform random choice
 	output_choice = Sample_memory[output_choice_index]      // k'
 	//fmt.Printf("Sample memory : %v \n in_memory: %v\n", Sample_memory, in_memory)
@@ -170,18 +190,19 @@ func (s *CMS) Merge(other *CMS) error {
 		return errors.New("countminsketch: matrix depth must match")
 	}
 
-	if s.k != other.s {
+	if s.k != other.k {
 		return errors.New("countminsketch: matrix width must match")
 	}
 
 	for i := uint(0); i < s.s; i++ {
 		for j := uint(0); j < s.k; j++ {
 			// s.mat[i][j] += other.mat[i][j] ADD
-			val := uint64((s.mat[i][j] + other.mat[i][j]) / 2) /* MOY */
-			if val < 1 {
+			var val float64
+			val = (float64(s.mat[i][j]) + float64(other.mat[i][j])) / 2 /* MOY */
+			if 0.0 < val && val < 1.0 {
 				s.mat[i][j] = 1
 			} else {
-				s.mat[i][j] = val
+				s.mat[i][j] = uint64(val)
 			}
 		}
 	}

scripts.sh

@@ -6,7 +6,7 @@ p="data/resultJul95"
 #p="data/resultess_1"
 for i in 2
 do
-    for j in {1..2}
+    for j in {3..5}
     do
         ./main -t=$i -n=$j -p=$p
     done

test.go

@@ -52,7 +52,7 @@ func test0() {
 
 	d := 50
 	w := 40
-	network.C = 10 //int(d)
+	network.C = 4 //int(d)
 	network.PeerCMS = network.InitCMS(uint(d), uint(w))
 	//fmt.Println(network.PeerCMS.MatToString())
 
@@ -62,7 +62,7 @@ func test0() {
 	//var listpeers = []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"}
 
 	var output = []string{}
-	fmt.Println("Memory", network.Sample_memory)
+	fmt.Println("Memory", network.Sample_memory, network.C)
 
 	for i, elm := range listpeers {
 		println(">>>>>>>>>>>>> ELEMENT", i+1, " ", elm)
@@ -106,8 +106,8 @@ func test_Knowledge_free() {
 	n := network.Read_occurence(listpeers)
 	/* CMS Parameters */
 
-	//k := int(math.Ceil(0.01 * float64(n))) // other test to do
-	k := int(math.Ceil(math.Log(float64(n)))) // round to the next integer
+	k := int(math.Ceil(0.01 * float64(n))) // other test to do
+	//k := int(math.Ceil(math.Log(float64(n)))) // round to the next integer
 	s := 10
 	network.C = 300
 
@@ -253,3 +253,61 @@ func check_cms() {
 
 	return
 }
+
+func test_Knowledge_free_with_small_set() {
+
+	path := input_path
+	var listpeers, err = readLines(path)
+	if err != nil {
+		log.Println("(check) Unable to read config file ", path)
+		return
+	}
+
+	m := len(listpeers)
+	log.Println("Input of size ", m)
+
+	n := network.Read_occurence(listpeers)
+	/* CMS Parameters */
+
+	k := int(math.Ceil(0.01 * float64(n))) // other test to do
+	//k := int(math.Ceil(math.Log(float64(n)))) // round to the next integer
+	s := 10
+	network.C = 300
+
+	network.PeerCMS = network.InitCMS(uint(s), uint(k))
+
+	fmt.Println(network.PeerCMS.MatToString())
+
+	/*  Knowledge free algorithm */
+	var output = []string{}
+	trunk := 1000
+	train := 100000
+	begin := m - trunk - train
+	fmt.Printf("Read %d elements between from element %d \n", trunk, begin)
+	for _, elm := range listpeers[begin : m-trunk] {
+		//println(">elmt", i)
+		network.PeerCMS.UpdateString(elm, 1)
+
+	}
+	for _, elm := range listpeers[m-trunk:] { //only trunk elements in the output
+		//println(">elmt", i)
+		network.PeerCMS.UpdateString(elm, 1)
+
+		value := network.PeerCMS.Knowledge_free(elm)
+		output = append(output, value)
+
+	}
+	fmt.Println(network.PeerCMS.MatToString())
+	/* summary */
+	fmt.Printf("A matrix of size %d*%d with sample memory length of %d\n Output of size %d\n",
+		s, k, len(network.Sample_memory), len(output))
+
+	out_path := "data/output" + strconv.Itoa(num_expe) // path of the unbiaised output stream
+	err = writeLines(output, out_path)
+	if err != nil {
+		log.Println("(Write) Unable to read config file ", out_path)
+		return
+	}
+
+	return
+}