-
Notifications
You must be signed in to change notification settings - Fork 24.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Extract device type from user agent info (#69322)
- Loading branch information
Showing
16 changed files
with
1,043 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
179 changes: 179 additions & 0 deletions
179
.../ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,179 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.ingest.useragent; | ||
|
|
||
| import org.elasticsearch.ElasticsearchParseException; | ||
| import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; | ||
| import org.elasticsearch.common.xcontent.NamedXContentRegistry; | ||
| import org.elasticsearch.common.xcontent.XContentFactory; | ||
| import org.elasticsearch.common.xcontent.XContentParser; | ||
| import org.elasticsearch.common.xcontent.XContentType; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.regex.Matcher; | ||
| import java.util.regex.Pattern; | ||
|
|
||
| import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations; | ||
| import static org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; | ||
|
|
||
| public class DeviceTypeParser { | ||
|
|
||
| private static final String OS_PARSERS = "os_parsers"; | ||
| private static final String BROWSER_PARSER = "browser_parsers"; | ||
| private static final String DEVICE_PARSER = "device_parsers"; | ||
| private static final String AGENT_STRING_PARSER = "agent_string_parsers"; | ||
| private static final String robot = "Robot", tablet = "Tablet", desktop = "Desktop", phone = "Phone"; | ||
|
|
||
| private final List<String> patternListKeys = List.of(OS_PARSERS, BROWSER_PARSER, DEVICE_PARSER, AGENT_STRING_PARSER); | ||
|
|
||
| private final HashMap<String, ArrayList<DeviceTypeSubPattern>> deviceTypePatterns = new HashMap<>(); | ||
|
|
||
| public void init(InputStream regexStream) throws IOException { | ||
| // EMPTY is safe here because we don't use namedObject | ||
| XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, | ||
| LoggingDeprecationHandler.INSTANCE, regexStream); | ||
|
|
||
| XContentParser.Token token = yamlParser.nextToken(); | ||
|
|
||
| if (token == XContentParser.Token.START_OBJECT) { | ||
| token = yamlParser.nextToken(); | ||
|
|
||
| for (; token != null; token = yamlParser.nextToken()) { | ||
| String currentName = yamlParser.currentName(); | ||
| if (token == XContentParser.Token.FIELD_NAME && patternListKeys.contains(currentName)) { | ||
| List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser); | ||
| ArrayList<DeviceTypeSubPattern> subPatterns = new ArrayList<>(); | ||
| for (Map<String, String> map : parserConfigurations) { | ||
| subPatterns.add(new DeviceTypeSubPattern(Pattern.compile((map.get("regex"))), | ||
| map.get("replacement"))); | ||
| } | ||
| deviceTypePatterns.put(currentName, subPatterns); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if (patternListKeys.size() != deviceTypePatterns.size()) { | ||
| throw new ElasticsearchParseException("not a valid regular expression file"); | ||
| } | ||
| } | ||
|
|
||
| public String findDeviceType(String agentString, VersionedName userAgent, VersionedName os, VersionedName device) { | ||
| if (deviceTypePatterns.isEmpty()) { | ||
| return null; | ||
| } | ||
| if (agentString != null) { | ||
| String deviceType = findMatch(deviceTypePatterns.get(AGENT_STRING_PARSER), agentString); | ||
| if (deviceType != null) { | ||
| return deviceType; | ||
| } | ||
| } | ||
| return findDeviceType(userAgent, os, device); | ||
| } | ||
|
|
||
| public String findDeviceType(VersionedName userAgent, VersionedName os, VersionedName device) { | ||
|
|
||
| if (deviceTypePatterns.isEmpty()) { | ||
| return null; | ||
| } | ||
|
|
||
| ArrayList<String> extractedDeviceTypes = new ArrayList<>(); | ||
|
|
||
| for (String patternKey : patternListKeys) { | ||
| String deviceType = null; | ||
| switch (patternKey) { | ||
| case OS_PARSERS: | ||
| if (os != null && os.name != null) { | ||
| deviceType = findMatch(deviceTypePatterns.get(patternKey), os.name); | ||
| } | ||
| break; | ||
| case BROWSER_PARSER: | ||
| if (userAgent != null && userAgent.name != null) { | ||
| deviceType = findMatch(deviceTypePatterns.get(patternKey), userAgent.name); | ||
| } | ||
| break; | ||
| case DEVICE_PARSER: | ||
| if (device != null && device.name != null) { | ||
| deviceType = findMatch(deviceTypePatterns.get(patternKey), device.name); | ||
| } | ||
| break; | ||
| default: | ||
| break; | ||
| } | ||
|
|
||
| if (deviceType != null) { | ||
| extractedDeviceTypes.add(deviceType); | ||
| } | ||
| } | ||
|
|
||
|
|
||
| if (extractedDeviceTypes.contains(robot)) { | ||
| return robot; | ||
| } | ||
| if (extractedDeviceTypes.contains(tablet)) { | ||
| return tablet; | ||
| } | ||
| if (extractedDeviceTypes.contains(phone)) { | ||
| return phone; | ||
| } | ||
| if (extractedDeviceTypes.contains(desktop)) { | ||
| return desktop; | ||
| } | ||
|
|
||
| return "Other"; | ||
| } | ||
|
|
||
| private String findMatch(List<DeviceTypeSubPattern> possiblePatterns, String matchString) { | ||
| String name; | ||
| for (DeviceTypeSubPattern pattern : possiblePatterns) { | ||
| name = pattern.match(matchString); | ||
| if (name != null) { | ||
| return name; | ||
| } | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| static final class DeviceTypeSubPattern { | ||
| private final Pattern pattern; | ||
| private final String nameReplacement; | ||
|
|
||
| DeviceTypeSubPattern(Pattern pattern, String nameReplacement) { | ||
| this.pattern = pattern; | ||
| this.nameReplacement = nameReplacement; | ||
| } | ||
|
|
||
| public String match(String matchString) { | ||
| String name = null; | ||
|
|
||
| Matcher matcher = pattern.matcher(matchString); | ||
|
|
||
| if (matcher.find() == false) { | ||
| return null; | ||
| } | ||
|
|
||
| int groupCount = matcher.groupCount(); | ||
|
|
||
| if (nameReplacement != null) { | ||
| if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { | ||
| name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); | ||
| } else { | ||
| name = nameReplacement; | ||
| } | ||
| } | ||
|
|
||
| return name; | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
67 changes: 67 additions & 0 deletions
67
modules/ingest-user-agent/src/main/resources/device_type_regexes.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| # Apache License, Version 2.0 | ||
| # =========================== | ||
| # | ||
| # Copyright 2009 Google Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| ## Custom parser being added to support device types | ||
|
|
||
| os_parsers: | ||
| # Robot | ||
| - regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google' | ||
| replacement: 'Robot' | ||
| # Desktop OS, Most Common | ||
| - regex: '^(Windows$|Windows NT$|Mac OS X|Linux$|Chrome OS|Fedora$|Ubuntu$)' | ||
| replacement: 'Desktop' | ||
| # Phone OS | ||
| - regex: '^(Android$|iOS|Windows Phone|Firefox OS|BlackBerry OS|KaiOS|Sailfish$|Maemo)' | ||
| replacement: 'Phone' | ||
| # Desktop OS, Not Common | ||
| - regex: '^(Windows XP|Windows 7|Windows 10|FreeBSD|OpenBSD|Arch Linux|Solaris|NetBSD|SUSE|SunOS|BeOS\/Haiku)' | ||
| replacement: 'Desktop' | ||
| - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' | ||
| replacement: 'Tablet' | ||
|
|
||
| browser_parsers: | ||
| # Robot | ||
| - regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google' | ||
| replacement: 'Robot' | ||
| # Desktop Browsers | ||
| - regex: '^(Chrome$|Chromium$|Edge$|Firefox$|IE$|Maxthon$|Opera$|Safari$|SeaMonkey$|Vivaldi$|Yandex Browser$)' | ||
| replacement: 'Desktop' | ||
| # Phone Browsers, Most Common | ||
| - regex: '^(Chrome Mobile$|Chrome Mobile iOS|Firefox Mobile|Firefox iOS|Edge Mobile|Android|Facebook|Instagram|IE Mobile)' | ||
| replacement: 'Phone' | ||
| # Phone Browsers, Not Common | ||
| - regex: '^(BlackBerry WebKit|OktaMobile|Sailfish Browser|Amazon Silk|Pinterest|Flipboard)' | ||
| replacement: 'Phone' | ||
| - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' | ||
| replacement: 'Tablet' | ||
|
|
||
| device_parsers: | ||
| - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk|Kindle' | ||
| replacement: 'Tablet' | ||
| # Samsung tablets | ||
| - regex: 'SM-T\d+|SM-P\d+|GT-P\d+' | ||
| replacement: 'Tablet' | ||
| # other tablets | ||
| - regex: 'Asus Nexus \d+|Lenovo TB' | ||
| replacement: 'Tablet' | ||
|
|
||
| agent_string_parsers: | ||
| - regex: 'Synthetic|Scanner|Crawler|Site24x7|PagePeeker|SpeedCurve|RuxitSynthetic|Google Web Preview|Synthetic|SiteChecker|Parser' | ||
| replacement: 'Robot' | ||
| - regex: 'Tablet' | ||
| replacement: 'Tablet' | ||
|
|
Oops, something went wrong.