Merge branch 'main' into mcp

Author: m-appel

.pre-commit-config.yaml

@@ -1,4 +1,5 @@
 files: \.py$
+
 repos:
 -   repo: https://github.com/PyCQA/autoflake
     rev: v2.3.1

ACKNOWLEDGMENTS.md

@@ -11,15 +11,27 @@ Please refer to the READMEs in the respective crawler directories for more infor
 
 We retrieve route server looking glass snapshots from the following IXPs.
 
-|   Name   |            URL             |
-|----------|----------------------------|
-| AMS-IX   | https://lg.ams-ix.net/     |
-| BCIX     | https://lg.bcix.de/        |
-| DE-CIX   | https://lg.de-cix.net/     |
-| IX.br    | https://lg.ix.br/          |
-| LINX     | https://alice-rs.linx.net/ |
-| Megaport | https://lg.megaport.com/   |
-| Netnod   | https://lg.netnod.se/      |
+|     Name     |            URL             |
+|--------------|----------------------------|
+| AMS-IX       | https://lg.ams-ix.net/     |
+| BCIX         | https://lg.bcix.de/        |
+| DD-IX        | https://lg.dd-ix.net       |
+| DE-CIX       | https://lg.de-cix.net/     |
+| IX.br        | https://lg.ix.br/          |
+| LINX         | https://alice-rs.linx.net/ |
+| Megaport     | https://lg.megaport.com/   |
+| Netnod       | https://lg.netnod.se/      |
+| IX Australia | https://lg.ix.asn.au       |
+| NZIX         | https://lg.ix.nz           |
+| PIX          | https://glass.gigapix.pt   |
+| SFMIX        | https://alice.sfmix.org    |
+| Stuttgart-IX | https://lg.s-ix.de         |
+| TOP-IX       | https://lg.top-ix.org      |
+
+## Amazon
+
+We use the [IP address ranges](https://ip-ranges.amazonaws.com/ip-ranges.json) provided by
+[Amazon Web Services](https://aws.amazon.com/).
 
 ## APNIC
 
@@ -117,6 +129,25 @@ Tech](https://inetintel.notion.site/Internet-Intelligence-Research-Lab-d18618456
 Use of this data is authorized under their [Acceptable Use
 Agreement](https://raw.githubusercontent.com/InetIntel/Dataset-AS-to-Organization-Mapping/master/LICENSE).
 
+## Internet Assigned Numbers Authority (IANA)
+
+We use multiple authoritative registries published by the
+[Internet Assigned Numbers Authority (IANA)](https://www.iana.org/) for modeling
+global Internet infrastructure.
+
+These datasets include:
+
+- The [DNS Root Zone File](https://www.iana.org/domains/root/files), used to retrieve
+  authoritative name servers and their IPv4 and IPv6 addresses for all top-level
+  domains.
+- The [IPv4 Address Space Registry](https://www.iana.org/assignments/ipv4-address-space/)
+  and the [IPv6 Unicast Address Assignments](https://www.iana.org/assignments/ipv6-unicast-address-assignments/),
+  used to model IANA-level address space allocations to Regional Internet Registries
+  (RIRs).
+- The [IPv4](https://www.iana.org/assignments/iana-ipv4-special-registry/) and
+  [IPv6](https://www.iana.org/assignments/iana-ipv6-special-registry/) Special-Purpose
+  Address Registries, used to identify reserved and protocol-specific address blocks.
+
 ## IPinfo
 
 We use the free [IP-to-country](https://ipinfo.io/products/free-ip-database) mapping

README.md

@@ -7,8 +7,9 @@ Internet resources (for example ASNs, IP prefixes, and domain names).
 
 Visit <https://iyp.iijlab.net> to try our online prototype. You will find instructions
 on how to connect to the prototype and some example queries there. For even more
-examples, check out the [IYP
-gallery](documentation/gallery.md).
+examples, check out the [IYP gallery](documentation/gallery.md). If you want to learn
+how to properly use IYP you should complete the [IYP
+tutorial](https://tutorial.iyp.ihr.live/).
 
 ## Deploy a local IYP instance
 
@@ -80,8 +81,9 @@ docker stop iyp
 
 ### Query the database
 
-Open <http://localhost:7474> in your favorite browser. To connect the interface to the database give
-the default login and password: `neo4j` and `password` respectively. Then enter your query in the top input field.
+Open <http://localhost:7474> in your favorite browser. No credentials are required to
+connect to the database, just click on “Connect”. Then enter your query in the top input
+field.
 
 For example, this finds the IXPs and corresponding country codes where IIJ (AS2497) is:
 

autodeploy/autodeploy.py

@@ -31,19 +31,21 @@ def remove_deployment(client: docker.DockerClient, date: datetime):
     try:
         container = client.containers.get(container_name)
         logging.warning(f'Removing active deployment for {date.strftime("%m-%d")}')
-        container.stop()
-        # Wait a little bit after the container has been removed
-        # before deleting the volume
-        # TODO Is there a better way?
-        while True:
-            try:
-                client.volumes.get(volume_name).remove()
-                break
-            except BaseException:
-                time.sleep(1)
+        container.stop(timeout=30)
+        container.wait(timeout=60)
     except docker.errors.NotFound:
         logging.info(f'No existing deployment for {date.strftime("%Y-%m-%d")}. Starting deployment')
 
+    for _ in range(10):
+        try:
+            client.volumes.get(volume_name).remove()
+            return
+        except docker.errors.NotFound:
+            return
+        except docker.errors.APIError:
+            time.sleep(1)
+    raise RuntimeError(f'Failed to remove volume {volume_name}')
+
 
 def get_ports_from_caddy_config(config: dict):
     """Makes a request to caddy config and returns the ports currently being used (both

basic/items.csv

@@ -91,17 +91,26 @@
             "iyp.crawlers.ripe.atlas_measurements",
             "iyp.crawlers.alice_lg.amsix",
             "iyp.crawlers.alice_lg.bcix",
+            "iyp.crawlers.alice_lg.ddix",
             "iyp.crawlers.alice_lg.decix",
+            "iyp.crawlers.alice_lg.ixaustralia",
             "iyp.crawlers.alice_lg.ixbr",
             "iyp.crawlers.alice_lg.linx",
             "iyp.crawlers.alice_lg.megaport",
             "iyp.crawlers.alice_lg.netnod",
+            "iyp.crawlers.alice_lg.nzix",
+            "iyp.crawlers.alice_lg.pix",
+            "iyp.crawlers.alice_lg.sfmix",
+            "iyp.crawlers.alice_lg.six",
+            "iyp.crawlers.alice_lg.topix",
             "iyp.crawlers.ipinfo.ip_country",
+            "iyp.crawlers.amazon.aws_ip_ranges",
             "iyp.crawlers.worldbank.country_pop",
             "iyp.crawlers.simulamet.rirdata_rdns",
-            "iyp.crawlers.ut_dacs.laces_v4",
-            "iyp.crawlers.ut_dacs.laces_v6",
+            "iyp.crawlers.utwente.laces_v4",
+            "iyp.crawlers.utwente.laces_v6",
             "iyp.crawlers.google.crux_top1m_country",
+            "iyp.crawlers.iana.address_space",
             "iyp.crawlers.openintel.crux",
             "iyp.crawlers.openintel.dnsgraph_nl",
             "iyp.crawlers.openintel.dnsgraph_rdns",
@@ -129,5 +138,12 @@
             "iyp.post.country_information",
             "iyp.post.url2hostname"
         ]
+    },
+
+    "neo4j": {
+        "server": "localhost",
+        "port": 7687,
+        "login": "neo4j",
+        "password": "password"
     }
 }