From 024efd1fc1af1737845690ed5bd17fe3088cc811 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Mon, 22 Jan 2024 10:30:31 +0200 Subject: [PATCH] Update ua_inspector database (#3698) * Update ua_inspector source * Downloan device detector updates --- config/config.exs | 2 +- mix.lock | 2 +- priv/ua_inspector/bot.bots.yml | 111 +- priv/ua_inspector/client.browsers.yml | 70 +- priv/ua_inspector/client.feed_readers.yml | 54 +- priv/ua_inspector/client.libraries.yml | 194 +- priv/ua_inspector/client.mediaplayers.yml | 57 + priv/ua_inspector/client.mobile_apps.yml | 827 +++++- priv/ua_inspector/client.pim.yml | 4 + priv/ua_inspector/client_hints.apps.yml | 17 + priv/ua_inspector/client_hints.browsers.yml | 22 + priv/ua_inspector/device.consoles.yml | 12 +- priv/ua_inspector/device.mobiles.yml | 2475 ++++++++++++----- .../device.portable_media_player.yml | 16 +- priv/ua_inspector/device.televisions.yml | 570 +++- priv/ua_inspector/os.oss.yml | 287 +- .../short_codes.browser_families.yml | 16 + .../short_codes.client_browsers.yml | 21 +- .../short_codes.mobile_browsers.yml | 14 + priv/ua_inspector/short_codes.os_families.yml | 6 + priv/ua_inspector/short_codes.oss.yml | 6 + priv/ua_inspector/ua_inspector.release | 2 +- 22 files changed, 3887 insertions(+), 898 deletions(-) diff --git a/config/config.exs b/config/config.exs index 65c37c127..a5b5e5eb9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -38,7 +38,7 @@ config :tailwind, config :ua_inspector, database_path: "priv/ua_inspector", - remote_release: "66d80de32fbb265941f4d7941fadc19097375097" + remote_release: "6.2.1" config :ref_inspector, database_path: "priv/ref_inspector" diff --git a/mix.lock b/mix.lock index b90b976e6..c5ff10fab 100644 --- a/mix.lock +++ b/mix.lock @@ -139,7 +139,7 @@ "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "tls_certificate_check": {:hex, :tls_certificate_check, "1.21.0", "042ab2c0c860652bc5cf69c94e3a31f96676d14682e22ec7813bd173ceff1788", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6cee6cffc35a390840d48d463541d50746a7b0e421acaadb833cfc7961e490e7"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, - "ua_inspector": {:hex, :ua_inspector, "3.7.0", "0cf79016e8da9d9f5598a9aec99d0812ee60a5dee78fc5340e443f336c704995", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "9ac3a0280c8e6060255357a3ce4fe9b4f0fd30780629dc0d6e008a03ab0e8f81"}, + "ua_inspector": {:hex, :ua_inspector, "3.8.0", "c0b0d13200a9bd509225f15ea8cf275c0bec27390a21c355746ff8b8a88c3e4d", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "7c980bae82a4754075b933e0f383935a681e5a2628856ad3ecf6eb80d8139539"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, diff --git a/priv/ua_inspector/bot.bots.yml b/priv/ua_inspector/bot.bots.yml index 33a0194fe..868bea9b9 100644 --- a/priv/ua_inspector/bot.bots.yml +++ b/priv/ua_inspector/bot.bots.yml @@ -19,7 +19,7 @@ url: 'https://developers.cloudflare.com/health-checks/' producer: name: 'CloudFlare' - url: 'http://www.cloudflare.com' + url: 'https://www.cloudflare.com/' - regex: '360Spider' name: '360Spider' @@ -331,21 +331,53 @@ name: 'CloudFlare' url: 'http://www.cloudflare.com' -- regex: 'CloudflareDiagnostics' +- regex: 'Cloudflare-?Diagnostics' name: 'Cloudflare Diagnostics' category: 'Site Monitor' url: 'https://www.cloudflare.com/' producer: name: 'Cloudflare' - url: 'https://www.cloudflare.com' + url: 'https://www.cloudflare.com/' - regex: 'CloudFlare-AlwaysOnline' name: 'CloudFlare Always Online' category: 'Site Monitor' - url: 'http://www.cloudflare.com/always-online' + url: 'https://www.cloudflare.com/always-online' producer: name: 'CloudFlare' - url: 'http://www.cloudflare.com' + url: 'https://www.cloudflare.com/' + +- regex: 'Cloudflare-SSLDetector' + name: 'Cloudflare SSL Detector' + category: 'Site Monitor' + url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/' + producer: + name: 'CloudFlare' + url: 'https://www.cloudflare.com/' + +- regex: 'Cloudflare Custom Hostname Verification' + name: 'Cloudflare Custom Hostname Verification' + category: 'Service Agent' + url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/' + producer: + name: 'CloudFlare' + url: 'https://www.cloudflare.com/' + +- regex: 'Cloudflare-Traffic-Manager' + name: 'Cloudflare Traffic Manager' + category: 'Site Monitor' + url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/' + producer: + name: 'CloudFlare' + url: 'https://www.cloudflare.com/' + +- regex: 'https://developers.cloudflare.com/security-center/' + name: 'Cloudflare Security Insights' + category: 'Site Monitor' + url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/' + producer: + name: 'CloudFlare' + url: 'https://www.cloudflare.com/' - regex: 'coccoc.com' name: 'Cốc Cốc Bot' @@ -2003,7 +2035,7 @@ - regex: 'RSSRadio \(Push Notification Scanner;support@dorada\.co\.uk\)' name: 'RSSRadio Bot' -- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?!(?: Build|Plus))|zeal(?!ot)|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|My User Agent|cortex|CF-UC User Agent|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|daumoa,damoa,daum,daumos,duamoa,duam,duamos|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|custom_user_agent|Test Certificate Info|iplabel|Magellan)' +- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?!(?: Build|Plus))|zeal(?!ot)|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|My User Agent|cortex|CF-UC User Agent|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|daumoa,damoa,daum,daumos,duamoa,duam,duamos|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|custom_user_agent|Test Certificate Info|iplabel|Magellan|CustomUserAgent)' name: 'Generic Bot' - regex: '^sentry' @@ -3435,6 +3467,71 @@ name: 'webwiki GmbH' url: 'https://www.webwiki.com/' +- regex: 'phpMyAdmin' + name: 'phpMyAdmin' + category: 'Service Agent' + url: 'https://www.phpmyadmin.net/' + +- regex: 'Matomo/([\d+.]+)' + name: 'Matomo' + category: 'Service Agent' + url: 'https://github.com/matomo-org/matomo' + producer: + name: 'InnoCraft Ltd' + url: 'https://matomo.org/' + +- regex: 'Prometheus/([\d+.]+)' + name: 'Prometheus' + category: 'Service Agent' + url: 'https://github.com/prometheus/prometheus' + producer: + name: 'The Linux Foundation' + url: 'https://www.cncf.io/' + +- regex: 'ArchiveTeam ArchiveBot' + name: 'ArchiveBot' + category: 'Crawler' + url: 'https://wiki.archiveteam.org/index.php?title=ArchiveBot' + producer: + name: 'ArchiveTeam' + url: 'https://wiki.archiveteam.org/' + +- regex: 'MADBbot/([\d+.]+)' + name: 'MADBbot' + category: 'Crawler' + url: 'https://madb.zapto.org/bot.html' + +- regex: 'MeltwaterNews' + name: 'MeltwaterNews' + category: 'Crawler' + producer: + name: 'Meltwater Deutschland GmbH' + url: 'https://www.meltwater.com/' + +- regex: '(?:Owler@ows.eu|OWLer)/([\d+.]+)' + name: 'OWLer' + category: 'Crawler' + url: 'https://openwebsearch.eu/owler/' + producer: + name: 'Open Search Foundation e.V.' + url: 'https://openwebsearch.eu/' + +- regex: 'bbc.co.uk/display/men/Page\+Monitor' + name: 'BBC Page Monitor' + category: 'Site Monitor' + url: 'https://confluence.dev.bbc.co.uk/display/men/Page+Monitor' + producer: + name: 'BBC' + url: 'https://www.bbc.com/' + +- regex: 'BBC-Forge-URL-Monitor-Twisted' + name: 'BBC Forge URL Monitor' + category: 'Site Monitor' + url: 'https://www.bbc.com/' + producer: + name: 'BBC' + url: 'https://www.bbc.com/' + # Generic detections -- regex: '[a-z0-9\-_]*((?