diff --git a/bin/run.sh b/bin/run.sh index c1a635f2fc..80910a127b 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -1,7 +1,8 @@ #!/bin/sh -java -cp `ls target/*-fatjar.jar` -Xms512M -Xmx192G -Dslf4j.internal.verbosity=WARN --add-modules jdk.incubator.vector $@ 2>&1 | grep -v "WARNING: Using incubator modules" +java -cp `ls target/*-fatjar.jar` -Xms512M -Xmx192G -Dslf4j.internal.verbosity=WARN --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED $@ 2>&1 | grep -v "WARNING: Using incubator modules" # Notes: # - "WARNING: Using incubator modules: jdk.incubator.vector" cannot be suppressed, so just grep -v it. # - "SLF4J(I): Connected with provider of type [org.apache.logging.slf4j.SLF4JServiceProvider]": suppress using -Dslf4j +# - "WARNING: A restricted method in java.lang.foreign.Linker has been called" fixed by --enable-native-access=ALL-UNNAMED diff --git a/docs/regressions/regressions-hc4-neuclir22-fa.md b/docs/regressions/regressions-hc4-neuclir22-fa.md index 42276e471e..98cb459440 100644 --- a/docs/regressions/regressions-hc4-neuclir22-fa.md +++ b/docs/regressions/regressions-hc4-neuclir22-fa.md @@ -156,21 +156,21 @@ With the above commands, you should be able to reproduce the following results: | **MAP** | **BM25 (default)**| **+RM3** | **+Rocchio**| |:-------------------------------------------------------------------------------------------------------------|-------------------|-----------|-------------| -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.1198 | 0.1064 | 0.1221 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.1435 | 0.0833 | 0.1254 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.1438 | 0.1075 | 0.1351 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.1331 | 0.1045 | 0.1318 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.1420 | 0.0998 | 0.1250 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.1536 | 0.1255 | 0.1382 | | **nDCG@20** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.1806 | 0.1564 | 0.1794 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.2288 | 0.1285 | 0.1968 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.2233 | 0.1706 | 0.2001 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.1999 | 0.1532 | 0.1847 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.2197 | 0.1526 | 0.1926 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.2371 | 0.1925 | 0.2057 | | **J@20** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.1430 | 0.1220 | 0.1520 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.1480 | 0.1110 | 0.1480 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.1570 | 0.1210 | 0.1530 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.1510 | 0.1270 | 0.1650 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.1490 | 0.1120 | 0.1580 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.1670 | 0.1270 | 0.1690 | | **Recall@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.7234 | 0.6742 | 0.7929 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.7431 | 0.6132 | 0.7768 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.7652 | 0.6436 | 0.8058 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.7483 | 0.6933 | 0.7991 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.7475 | 0.6365 | 0.8233 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.7964 | 0.6835 | 0.8532 | The above results reproduce the BM25 title queries run in Table 2 of [this paper](https://arxiv.org/pdf/2201.08471.pdf). diff --git a/docs/regressions/regressions-hc4-v1.0-fa.md b/docs/regressions/regressions-hc4-v1.0-fa.md index 9dc8631d70..446af21d51 100644 --- a/docs/regressions/regressions-hc4-v1.0-fa.md +++ b/docs/regressions/regressions-hc4-v1.0-fa.md @@ -245,33 +245,33 @@ With the above commands, you should be able to reproduce the following results: | **MAP** | **BM25 (default)**| **+RM3** | **+Rocchio**| |:-------------------------------------------------------------------------------------------------------------|-------------------|-----------|-------------| -| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.2971 | 0.2865 | 0.3030 | -| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.3243 | 0.3402 | 0.3721 | -| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.3366 | 0.2919 | 0.3881 | -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.2877 | 0.2954 | 0.2954 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.2928 | 0.2807 | 0.2928 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.3285 | 0.3042 | 0.3274 | +| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.2818 | 0.2583 | 0.2761 | +| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.3128 | 0.2984 | 0.3442 | +| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.3343 | 0.2680 | 0.3510 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.3059 | 0.3080 | 0.3182 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.3108 | 0.2714 | 0.3170 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.3437 | 0.3058 | 0.3507 | | **nDCG@20** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.3445 | 0.3446 | 0.3161 | -| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.3475 | 0.3862 | 0.3895 | -| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.3665 | 0.3454 | 0.3962 | -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.3846 | 0.3812 | 0.3861 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.4039 | 0.3755 | 0.3811 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.4489 | 0.4049 | 0.4256 | +| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.3315 | 0.3186 | 0.2806 | +| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.3394 | 0.3508 | 0.3599 | +| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.3664 | 0.3132 | 0.3490 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.4114 | 0.3982 | 0.3958 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.4077 | 0.3513 | 0.3942 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.4606 | 0.4006 | 0.4341 | | **J@20** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.4100 | 0.3300 | 0.3950 | -| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.4750 | 0.3500 | 0.5100 | -| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.4700 | 0.3450 | 0.5250 | -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.4010 | 0.3800 | 0.4350 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.3890 | 0.3590 | 0.4300 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.4420 | 0.3800 | 0.4580 | +| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.3900 | 0.3500 | 0.3650 | +| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.4600 | 0.3300 | 0.4650 | +| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.4600 | 0.2900 | 0.4800 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.4220 | 0.4030 | 0.4600 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.3950 | 0.3410 | 0.4570 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.4460 | 0.3980 | 0.4840 | | **Recall@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.7794 | 0.7683 | 0.8039 | -| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.8491 | 0.7737 | 0.8838 | -| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.9129 | 0.8102 | 0.9154 | -| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.8223 | 0.7755 | 0.8560 | -| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.8402 | 0.7487 | 0.8738 | -| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.8651 | 0.8055 | 0.8883 | +| [HC4 (Persian): dev-topic title](https://github.com/hltcoe/HC4) | 0.7745 | 0.7835 | 0.8039 | +| [HC4 (Persian): dev-topic description](https://github.com/hltcoe/HC4) | 0.8433 | 0.7685 | 0.8727 | +| [HC4 (Persian): dev-topic description+title](https://github.com/hltcoe/HC4) | 0.9128 | 0.8077 | 0.9323 | +| [HC4 (Persian): test-topic title](https://github.com/hltcoe/HC4) | 0.8405 | 0.7957 | 0.8721 | +| [HC4 (Persian): test-topic description](https://github.com/hltcoe/HC4) | 0.8581 | 0.7488 | 0.8853 | +| [HC4 (Persian): test-topic description+title](https://github.com/hltcoe/HC4) | 0.8799 | 0.8051 | 0.9035 | The above results reproduce the BM25 title queries run in Table 2 of [this paper](https://arxiv.org/pdf/2201.08471.pdf). diff --git a/docs/regressions/regressions-miracl-v1.0-fa-aca.md b/docs/regressions/regressions-miracl-v1.0-fa-aca.md index 8567cd4399..1117577305 100644 --- a/docs/regressions/regressions-miracl-v1.0-fa-aca.md +++ b/docs/regressions/regressions-miracl-v1.0-fa-aca.md @@ -57,6 +57,6 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BM25** | |:-------------------------------------------------------------------------------------------------------------|-----------| -| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.3427 | +| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.3576 | | **R@100** | **BM25** | -| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.7600 | +| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.7806 | diff --git a/docs/regressions/regressions-miracl-v1.0-fa.md b/docs/regressions/regressions-miracl-v1.0-fa.md index 0c3c76b5c8..3a01c0ecd4 100644 --- a/docs/regressions/regressions-miracl-v1.0-fa.md +++ b/docs/regressions/regressions-miracl-v1.0-fa.md @@ -55,6 +55,6 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BM25** | |:-------------------------------------------------------------------------------------------------------------|-----------| -| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.3334 | +| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.3568 | | **R@100** | **BM25** | -| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.7306 | +| [MIRACL (Persian): dev](https://github.com/project-miracl/miracl) | 0.7413 | diff --git a/docs/regressions/regressions-neuclir22-fa-qt.md b/docs/regressions/regressions-neuclir22-fa-qt.md index 60896d49fa..bb5cd8e73f 100644 --- a/docs/regressions/regressions-neuclir22-fa-qt.md +++ b/docs/regressions/regressions-neuclir22-fa-qt.md @@ -245,33 +245,33 @@ With the above commands, you should be able to reproduce the following results: | **MAP** | **BM25 (default)**| **+RM3** | **+Rocchio**| |:-------------------------------------------------------------------------------------------------------------|-------------------|-----------|-------------| -| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.2151 | 0.1640 | 0.2304 | -| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.1875 | 0.1070 | 0.2077 | -| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.2271 | 0.1408 | 0.2358 | -| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.2169 | 0.1767 | 0.2321 | -| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.1925 | 0.1227 | 0.2266 | -| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.2526 | 0.1681 | 0.2597 | +| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.2492 | 0.1791 | 0.2289 | +| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.1970 | 0.1182 | 0.2156 | +| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.2484 | 0.1511 | 0.2339 | +| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.2638 | 0.1876 | 0.2545 | +| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.1956 | 0.1196 | 0.2234 | +| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.2648 | 0.1715 | 0.2590 | | **nDCG@20** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.3428 | 0.2775 | 0.3356 | -| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.2843 | 0.2010 | 0.2885 | -| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.3429 | 0.2463 | 0.3408 | -| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.3331 | 0.2917 | 0.3374 | -| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.2974 | 0.2146 | 0.3300 | -| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.3700 | 0.2963 | 0.3612 | +| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.3595 | 0.2792 | 0.3284 | +| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.2966 | 0.2137 | 0.2938 | +| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.3488 | 0.2457 | 0.3237 | +| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.3678 | 0.2876 | 0.3619 | +| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.3011 | 0.2168 | 0.3043 | +| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.3758 | 0.2793 | 0.3496 | | **J@20** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.3759 | 0.3250 | 0.3882 | -| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.3592 | 0.2684 | 0.3781 | -| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.4035 | 0.3110 | 0.4035 | -| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.3684 | 0.3307 | 0.3899 | -| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.3412 | 0.2592 | 0.3618 | -| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.3820 | 0.3110 | 0.3908 | +| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.3746 | 0.3197 | 0.3912 | +| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.3496 | 0.2654 | 0.3820 | +| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.3873 | 0.3057 | 0.3991 | +| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.3789 | 0.3355 | 0.3908 | +| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.3487 | 0.2662 | 0.3754 | +| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.3873 | 0.3175 | 0.3956 | | **Recall@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.7128 | 0.6870 | 0.7691 | -| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.7027 | 0.5572 | 0.7520 | -| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.7373 | 0.6414 | 0.8092 | -| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.7254 | 0.6877 | 0.7672 | -| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.6815 | 0.5606 | 0.7033 | -| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.7424 | 0.6264 | 0.7829 | +| [NeuCLIR 2022 (Persian): title (human-translated queries)](https://neuclir.github.io/) | 0.7324 | 0.6940 | 0.7928 | +| [NeuCLIR 2022 (Persian): desc (human-translated queries)](https://neuclir.github.io/) | 0.7214 | 0.5598 | 0.7728 | +| [NeuCLIR 2022 (Persian): desc+title (human-translated queries)](https://neuclir.github.io/) | 0.7494 | 0.6383 | 0.8294 | +| [NeuCLIR 2022 (Persian): title (machine-translated queries)](https://neuclir.github.io/) | 0.7631 | 0.7070 | 0.8137 | +| [NeuCLIR 2022 (Persian): desc (machine-translated queries)](https://neuclir.github.io/) | 0.6847 | 0.5597 | 0.7685 | +| [NeuCLIR 2022 (Persian): desc+title (machine-translated queries)](https://neuclir.github.io/) | 0.7714 | 0.6501 | 0.8227 | ## Reproduction Log[*](../../docs/reproducibility.md) diff --git a/src/main/resources/regression/hc4-neuclir22-fa.yaml b/src/main/resources/regression/hc4-neuclir22-fa.yaml index 32abfd501d..e41ab128d6 100644 --- a/src/main/resources/regression/hc4-neuclir22-fa.yaml +++ b/src/main/resources/regression/hc4-neuclir22-fa.yaml @@ -62,58 +62,58 @@ models: params: -bm25 -language fa results: MAP: - - 0.1198 - - 0.1435 - - 0.1438 + - 0.1331 + - 0.1420 + - 0.1536 nDCG@20: - - 0.1806 - - 0.2288 - - 0.2233 + - 0.1999 + - 0.2197 + - 0.2371 J@20: - - 0.1430 - - 0.1480 - - 0.1570 + - 0.1510 + - 0.1490 + - 0.1670 Recall@1000: - - 0.7234 - - 0.7431 - - 0.7652 + - 0.7483 + - 0.7475 + - 0.7964 - name: bm25-default+rm3 display: +RM3 params: -bm25 -rm3 -language fa results: MAP: - - 0.1064 - - 0.0833 - - 0.1075 + - 0.1045 + - 0.0998 + - 0.1255 nDCG@20: - - 0.1564 - - 0.1285 - - 0.1706 + - 0.1532 + - 0.1526 + - 0.1925 J@20: - - 0.1220 - - 0.1110 - - 0.1210 + - 0.1270 + - 0.1120 + - 0.1270 Recall@1000: - - 0.6742 - - 0.6132 - - 0.6436 + - 0.6933 + - 0.6365 + - 0.6835 - name: bm25-default+rocchio display: +Rocchio params: -bm25 -rocchio -language fa results: MAP: - - 0.1221 - - 0.1254 - - 0.1351 + - 0.1318 + - 0.1250 + - 0.1382 nDCG@20: - - 0.1794 - - 0.1968 - - 0.2001 + - 0.1847 + - 0.1926 + - 0.2057 J@20: - - 0.1520 - - 0.1480 - - 0.1530 + - 0.1650 + - 0.1580 + - 0.1690 Recall@1000: - - 0.7929 - - 0.7768 - - 0.8058 + - 0.7991 + - 0.8233 + - 0.8532 diff --git a/src/main/resources/regression/hc4-v1.0-fa.yaml b/src/main/resources/regression/hc4-v1.0-fa.yaml index 7d1ad13d5e..79be730821 100644 --- a/src/main/resources/regression/hc4-v1.0-fa.yaml +++ b/src/main/resources/regression/hc4-v1.0-fa.yaml @@ -75,94 +75,94 @@ models: params: -bm25 -language fa results: MAP: - - 0.2971 - - 0.3243 - - 0.3366 - - 0.2877 - - 0.2928 - - 0.3285 + - 0.2818 + - 0.3128 + - 0.3343 + - 0.3059 + - 0.3108 + - 0.3437 nDCG@20: - - 0.3445 - - 0.3475 - - 0.3665 - - 0.3846 - - 0.4039 - - 0.4489 + - 0.3315 + - 0.3394 + - 0.3664 + - 0.4114 + - 0.4077 + - 0.4606 J@20: - - 0.4100 - - 0.4750 - - 0.4700 - - 0.4010 - - 0.3890 - - 0.4420 + - 0.3900 + - 0.4600 + - 0.4600 + - 0.4220 + - 0.3950 + - 0.4460 Recall@1000: - - 0.7794 - - 0.8491 - - 0.9129 - - 0.8223 - - 0.8402 - - 0.8651 + - 0.7745 + - 0.8433 + - 0.9128 + - 0.8405 + - 0.8581 + - 0.8799 - name: bm25-default+rm3 display: +RM3 params: -bm25 -rm3 -language fa results: MAP: - - 0.2865 - - 0.3402 - - 0.2919 - - 0.2954 - - 0.2807 - - 0.3042 + - 0.2583 + - 0.2984 + - 0.2680 + - 0.3080 + - 0.2714 + - 0.3058 nDCG@20: - - 0.3446 - - 0.3862 - - 0.3454 - - 0.3812 - - 0.3755 - - 0.4049 + - 0.3186 + - 0.3508 + - 0.3132 + - 0.3982 + - 0.3513 + - 0.4006 J@20: - - 0.3300 - 0.3500 - - 0.3450 - - 0.3800 - - 0.3590 - - 0.3800 + - 0.3300 + - 0.2900 + - 0.4030 + - 0.3410 + - 0.3980 Recall@1000: - - 0.7683 - - 0.7737 - - 0.8102 - - 0.7755 - - 0.7487 - - 0.8055 + - 0.7835 + - 0.7685 + - 0.8077 + - 0.7957 + - 0.7488 + - 0.8051 - name: bm25-default+rocchio display: +Rocchio params: -bm25 -rocchio -language fa results: MAP: - - 0.3030 - - 0.3721 - - 0.3881 - - 0.2954 - - 0.2928 - - 0.3274 + - 0.2761 + - 0.3442 + - 0.3510 + - 0.3182 + - 0.3170 + - 0.3507 nDCG@20: - - 0.3161 - - 0.3895 - - 0.3962 - - 0.3861 - - 0.3811 - - 0.4256 + - 0.2806 + - 0.3599 + - 0.3490 + - 0.3958 + - 0.3942 + - 0.4341 J@20: - - 0.3950 - - 0.5100 - - 0.5250 - - 0.4350 - - 0.4300 - - 0.4580 + - 0.3650 + - 0.4650 + - 0.4800 + - 0.4600 + - 0.4570 + - 0.4840 Recall@1000: - 0.8039 - - 0.8838 - - 0.9154 - - 0.8560 - - 0.8738 - - 0.8883 + - 0.8727 + - 0.9323 + - 0.8721 + - 0.8853 + - 0.9035 diff --git a/src/main/resources/regression/miracl-v1.0-fa-aca.yaml b/src/main/resources/regression/miracl-v1.0-fa-aca.yaml index cc88fe16a2..af79f8988e 100644 --- a/src/main/resources/regression/miracl-v1.0-fa-aca.yaml +++ b/src/main/resources/regression/miracl-v1.0-fa-aca.yaml @@ -41,6 +41,6 @@ models: params: -bm25 -hits 100 -language fa -useAutoCompositeAnalyzer results: nDCG@10: - - 0.3427 + - 0.3576 R@100: - - 0.7600 + - 0.7806 diff --git a/src/main/resources/regression/miracl-v1.0-fa.yaml b/src/main/resources/regression/miracl-v1.0-fa.yaml index 088384b4d5..9d26130436 100644 --- a/src/main/resources/regression/miracl-v1.0-fa.yaml +++ b/src/main/resources/regression/miracl-v1.0-fa.yaml @@ -41,6 +41,6 @@ models: params: -bm25 -hits 100 -language fa results: nDCG@10: - - 0.3334 + - 0.3568 R@100: - - 0.7306 + - 0.7413 diff --git a/src/main/resources/regression/neuclir22-fa-qt.yaml b/src/main/resources/regression/neuclir22-fa-qt.yaml index ef1fca45d2..213bc2fd55 100644 --- a/src/main/resources/regression/neuclir22-fa-qt.yaml +++ b/src/main/resources/regression/neuclir22-fa-qt.yaml @@ -74,94 +74,94 @@ models: params: -bm25 -language fa results: MAP: - - 0.2151 - - 0.1875 - - 0.2271 - - 0.2169 - - 0.1925 - - 0.2526 + - 0.2492 + - 0.1970 + - 0.2484 + - 0.2638 + - 0.1956 + - 0.2648 nDCG@20: - - 0.3428 - - 0.2843 - - 0.3429 - - 0.3331 - - 0.2974 - - 0.3700 + - 0.3595 + - 0.2966 + - 0.3488 + - 0.3678 + - 0.3011 + - 0.3758 J@20: - - 0.3759 - - 0.3592 - - 0.4035 - - 0.3684 - - 0.3412 - - 0.3820 + - 0.3746 + - 0.3496 + - 0.3873 + - 0.3789 + - 0.3487 + - 0.3873 Recall@1000: - - 0.7128 - - 0.7027 - - 0.7373 - - 0.7254 - - 0.6815 - - 0.7424 + - 0.7324 + - 0.7214 + - 0.7494 + - 0.7631 + - 0.6847 + - 0.7714 - name: bm25-default+rm3 display: +RM3 params: -bm25 -rm3 -language fa results: MAP: - - 0.1640 - - 0.1070 - - 0.1408 - - 0.1767 - - 0.1227 - - 0.1681 + - 0.1791 + - 0.1182 + - 0.1511 + - 0.1876 + - 0.1196 + - 0.1715 nDCG@20: - - 0.2775 - - 0.2010 - - 0.2463 - - 0.2917 - - 0.2146 - - 0.2963 + - 0.2792 + - 0.2137 + - 0.2457 + - 0.2876 + - 0.2168 + - 0.2793 J@20: - - 0.3250 - - 0.2684 - - 0.3110 - - 0.3307 - - 0.2592 - - 0.3110 + - 0.3197 + - 0.2654 + - 0.3057 + - 0.3355 + - 0.2662 + - 0.3175 Recall@1000: - - 0.6870 - - 0.5572 - - 0.6414 - - 0.6877 - - 0.5606 - - 0.6264 + - 0.6940 + - 0.5598 + - 0.6383 + - 0.7070 + - 0.5597 + - 0.6501 - name: bm25-default+rocchio display: +Rocchio params: -bm25 -rocchio -language fa results: MAP: - - 0.2304 - - 0.2077 - - 0.2358 - - 0.2321 - - 0.2266 - - 0.2597 + - 0.2289 + - 0.2156 + - 0.2339 + - 0.2545 + - 0.2234 + - 0.2590 nDCG@20: - - 0.3356 - - 0.2885 - - 0.3408 - - 0.3374 - - 0.3300 - - 0.3612 + - 0.3284 + - 0.2938 + - 0.3237 + - 0.3619 + - 0.3043 + - 0.3496 J@20: - - 0.3882 - - 0.3781 - - 0.4035 - - 0.3899 - - 0.3618 + - 0.3912 + - 0.3820 + - 0.3991 - 0.3908 + - 0.3754 + - 0.3956 Recall@1000: - - 0.7691 - - 0.7520 - - 0.8092 - - 0.7672 - - 0.7033 - - 0.7829 + - 0.7928 + - 0.7728 + - 0.8294 + - 0.8137 + - 0.7685 + - 0.8227