// Copyright 1999 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// [Translated from] File: robots.cc
// -----------------------------------------------------------------------------
//
// Implements expired internet draft
//   http://www.robotstxt.org/norobots-rfc.txt
// with Google-specific optimizations detailed at
//   https://developers.google.com/search/reference/robots_txt

module unrobotstxt;

@safe:

import std.algorithm;
import std.array;
import std.ascii;
import std.container.array;
import std.conv;
import std.exception;
import std.range;
import std.typecons : Rebindable;
import std.utf;

/// Handler for directives found in robots.txt. These callbacks are called by
/// ParseRobotsTxt() in the sequence they have been found in the file.
abstract class RobotsParseHandler
{
public:

	void HandleRobotsStart();
	void HandleRobotsEnd();

	void HandleUserAgent(int line_num, string value);
	void HandleAllow(int line_num, string value);
	void HandleDisallow(int line_num, string value);

	void HandleSitemap(int line_num, string value);

	/// Any other unrecognized name/value pairs.
	void HandleUnknownAction(int line_num, string action, string value);
}

/// Parses body of a robots.txt and emits parse callbacks. This will accept
/// typical typos found in robots.txt, such as 'disalow'.
///
/// Note, this function will accept all kind of input but will skip
/// everything that does not look like a robots directive.
void ParseRobotsTxt(string robots_body, RobotsParseHandler parse_callback)
{
	auto parser = RobotsTxtParser(robots_body, parse_callback);
	parser.Parse();
}

final class RobotsMatcher : RobotsParseHandler
{
public:

	/// Create a RobotsMatcher with the default matching strategy. The default
	/// matching strategy is longest-match as opposed to the former internet draft
	/// that provisioned first-match strategy. Analysis shows that longest-match,
	/// while more restrictive for crawlers, is what webmasters assume when writing
	/// directives. For example, in case of conflicting matches (both Allow and
	/// Disallow), the longest match is the one the user wants. For example, in
	/// case of a robots.txt file that has the following rules
	///   Allow: /
	///   Disallow: /cgi-bin
	/// it's pretty obvious what the webmaster wants: they want to allow crawl of
	/// every URI except /cgi-bin. However, according to the expired internet
	/// standard, crawlers should be allowed to crawl everything with such a rule.
	this()
	{
		match_strategy_ = new LongestMatchRobotsMatchStrategy();
	}

	/// Verifies that the given user agent is valid to be matched against
	/// robots.txt. Valid user agent strings only contain the characters
	/// [a-zA-Z_-].
	static bool IsValidUserAgentToObey(string user_agent)
	{
		return !user_agent.empty && extractUserAgent(user_agent) == user_agent;
	}

	/// Returns true iff 'url' is allowed to be fetched by any member of the
	/// "user_agents" vector. 'url' must be %-encoded according to RFC3986.
	bool AllowedByRobots(string robots_body, const(string[]) user_agents, string url)
	{
		string path = GetPathParamsQuery(url);
		InitUserAgentsAndPath(user_agents, path);
		ParseRobotsTxt(robots_body, this);
		return !disallow();
	}

	/// Do robots check for 'url' when there is only one user agent. 'url' must
	/// be %-encoded according to RFC3986.
	bool OneAgentAllowedByRobots(string robots_txt, string user_agent, string url)
	{
		string[] v = [user_agent];
		return AllowedByRobots(robots_txt, v, url);
	}

	/// Returns true if we are disallowed from crawling a matching URI.
	bool disallow() const
	{
		if (allow_.specific.priority() > 0 || disallow_.specific.priority() > 0)
		{
			return (disallow_.specific.priority() > allow_.specific.priority());
		}

		if (ever_seen_specific_agent_)
		{
			// Matching group for user-agent but either without disallow or empty one,
			// i.e. priority == 0.
			return false;
		}

		if (disallow_.global.priority() > 0 || allow_.global.priority() > 0)
		{
			return disallow_.global.priority() > allow_.global.priority();
		}
		return false;
	}

	/// Returns true if we are disallowed from crawling a matching URI. Ignores any
	/// rules specified for the default user agent, and bases its results only on
	/// the specified user agents.
	bool disallow_ignore_global() const
	{
		if (allow_.specific.priority() > 0 || disallow_.specific.priority() > 0)
		{
			return disallow_.specific.priority() > allow_.specific.priority();
		}
		return false;
	}

	/// Returns true iff, when AllowedByRobots() was called, the robots file
	/// referred explicitly to one of the specified user agents.
	bool ever_seen_specific_agent() const
	{
		return ever_seen_specific_agent_;
	}

	/// Returns the line that matched or 0 if none matched.
	int matching_line() const
	{
		if (ever_seen_specific_agent_)
		{
			return Match.HigherPriorityMatch(&disallow_.specific, &allow_.specific).line();
		}
		return Match.HigherPriorityMatch(&disallow_.global, &allow_.global).line();
	}

protected:

	/// Parse callbacks.
	/// Protected because used in unittest. Never override RobotsMatcher, implement
	/// googlebot::RobotsParseHandler instead.
	override void HandleRobotsStart()
	{
		// This is a new robots.txt file, so we need to reset all the instance member
		// variables. We do it in the same order the instance member variables are
		// declared, so it's easier to keep track of which ones we have (or maybe
		// haven't!) done.
		allow_.Clear();
		disallow_.Clear();

		seen_global_agent_ = false;
		seen_specific_agent_ = false;
		ever_seen_specific_agent_ = false;
		seen_separator_ = false;
	}

	override void HandleRobotsEnd()
	{
	}

	override void HandleUserAgent(int line_num, string user_agent)
	{
		if (seen_separator_)
		{
			seen_specific_agent_ = seen_global_agent_ = seen_separator_ = false;
		}

		// Google-specific optimization: a '*' followed by space and more characters
		// in a user-agent record is still regarded a global rule.
		if (user_agent.length >= 1 && user_agent[0] == '*'
				&& (user_agent.length == 1 || std.ascii.isWhite(user_agent[1])))
		{
			seen_global_agent_ = true;
		}
		else
		{
			user_agent = extractUserAgent(user_agent);
			foreach (agent; user_agents_)
			{
				if (equalsIgnoreAsciiCase(user_agent, agent))
				{
					ever_seen_specific_agent_ = seen_specific_agent_ = true;
					break;
				}
			}
		}
	}

	override void HandleAllow(int line_num, string value)
	{
		if (!seen_any_agent) return;
		seen_separator_ = true;
		const int priority = match_strategy_.MatchAllow(path_, value);
		if (priority >= 0)
		{
			if (seen_specific_agent_)
			{
				if (allow_.specific.priority() < priority)
				{
					allow_.specific.Set(priority, line_num);
				}
			}
			else
			{
				assert (seen_global_agent_);
				if (allow_.global.priority() < priority)
				{
					allow_.global.Set(priority, line_num);
				}
			}
		}
		else
		{
			// Google-specific optimization: 'index.htm' and 'index.html' are normalized
			// to '/'.
			if (value.empty) return;
			auto last_part = value.byCodeUnit.splitter('/').tail(1).front;
			if (last_part.startsWith("index.htm"))
			{
				auto new_pattern = value[0 .. $ - last_part.length] ~ '$';
				HandleAllow(line_num, new_pattern);
			}
		}
	}

	override void HandleDisallow(int line_num, string value)
	{
		if (!seen_any_agent) return;
		seen_separator_ = true;
		const int priority = match_strategy_.MatchDisallow(path_, value);
		if (priority >= 0)
		{
			if (seen_specific_agent_)
			{
				if (disallow_.specific.priority() < priority)
				{
					disallow_.specific.Set(priority, line_num);
				}
			}
			else
			{
				assert (seen_global_agent_);
				if (disallow_.global.priority() < priority)
				{
					disallow_.global.Set(priority, line_num);
				}
			}
		}
	}

	override void HandleSitemap(int line_num, string value)
	{
		seen_separator_ = true;
	}

	override void HandleUnknownAction(int line_num, string action, string value)
	{
		seen_separator_ = true;
	}

package:

	/// Initialize next path and user-agents to check. Path must contain only the
	/// path, params, and query (if any) of the url and must start with a '/'.
	void InitUserAgentsAndPath(const(string)[] user_agents, string path)
	{
		path_ = path;
		assert (path_.startsWith('/'));
		user_agents_ = user_agents;
	}

	/// Returns true if any user-agent was seen.
	bool seen_any_agent() const
	{
		return seen_global_agent_ || seen_specific_agent_;
	}

	/// Instead of just maintaining a Boolean indicating whether a given line has
	/// matched, we maintain a count of the maximum number of characters matched by
	/// that pattern.
	///
	/// This structure stores the information associated with a match (e.g. when a
	/// Disallow is matched) as priority of the match and line matching.
	///
	/// The priority is initialized with a negative value to make sure that a match
	/// of priority 0 is higher priority than no match at all.
	struct Match
	{
	private:

		enum kNoMatchPriority = -1;

	public:

		void Set(int priority, int line)
		{
			priority_ = priority;
			line_ = line;
		}

		void Clear()
		{
			Set(kNoMatchPriority, 0);
		}

		int line() const
		{
			return line_;
		}

		int priority() const
		{
			return priority_;
		}

		static const(Match)* HigherPriorityMatch(const(Match)* a, const(Match)* b)
		{
			if (a.priority > b.priority)
			{
				return a;
			}
			else
			{
				return b;
			}
		}

	private:

		int priority_ = kNoMatchPriority;
		int line_ = 0;
	}

	/// For each of the directives within user-agents, we keep global and specific
	/// match scores.
	struct MatchHierarchy
	{
		/// Match for '*'
		Match global;
		/// Match for queried agent.
		Match specific;
		void Clear()
		{
			global.Clear();
			specific.Clear();
		}
	}

	/// Characters of 'url' matching Allow.
	MatchHierarchy allow_;
	/// Characters of 'url' matching Disallow.
	MatchHierarchy disallow_;

	/// True if processing global agent rules.
	bool seen_global_agent_ = false;
	/// True if processing our specific agent.
	bool seen_specific_agent_ = false;
	/// True if we ever saw a block for our agent.
	bool ever_seen_specific_agent_ = false;
	/// True if saw any key: value pair.
	bool seen_separator_ = false;

	/// The path we want to pattern match. Not owned and only a valid pointer
	/// during the lifetime of *AllowedByRobots calls.
	string path_;
	// The User-Agents we are interested in. Not owned and only a valid
	// pointer during the lifetime of *AllowedByRobots calls.
	const(string)[] user_agents_;

	RobotsMatchStrategy match_strategy_;
}

/// A RobotsMatchStrategy defines a strategy for matching individual lines in a
/// robots.txt file. Each Match* method should return a match priority, which is
/// interpreted as:
///
/// match priority < 0:
///    No match.
///
/// match priority == 0:
///    Match, but treat it as if matched an empty pattern.
///
/// match priority > 0:
///    Match.
abstract class RobotsMatchStrategy
{
public:

	int MatchAllow(string path, string pattern);
	int MatchDisallow(string path, string pattern);
}

/// Extracts path (with params) and query part from URL. Removes scheme,
/// authority, and fragment. Result always starts with "/".
/// Returns "/" if the url doesn't have a path or is not valid.
string GetPathParamsQuery(string url)
{
	auto remainder = url.byCodeUnit;

	// Initial two slashes are ignored.
	if (url.startsWith("//")) remainder = remainder[2 .. $];

	// Find end of protocol, if it exists, and remove it plus following ://
    // If path, param or query starts before ://, :// doesn't indicate protocol.
	auto protocol_point = remainder.findAmong(":/?;");
	while (protocol_point.startsWith(":"))
	{
		if (protocol_point[1 .. $].startsWith("//"))
		{
			remainder = protocol_point[3 .. $];
			break;
		}
		protocol_point = protocol_point[1 .. $];
	}

	auto path = remainder.findAmong("/?;");
	if (path.empty) return "/";
	auto hash = remainder.find('#');
	if (hash.length > path.length) return "/";

	path = path[0 .. $ - hash.length];

	if (path[0] != '/') return '/' ~ path.source;
	return path.source;
}

package immutable kHexDigits = "0123456789ABCDEF";

/// Canonicalize the allowed/disallowed paths. For example:
///     /SanJoséSellers ==> /Sanjos%C3%A9Sellers
///     %aa ==> %AA
string MaybeEscapePattern(string src)
{
	int num_to_escape = 0;
	bool need_capitalize = false;

	// First, scan the buffer to see if changes are needed. Most don't.
	foreach (i; 0 .. src.length)
	{
		// (a) % escape sequence.
		if (i + 2 < src.length && src[i] == '%'
				&& std.ascii.isHexDigit(src[i + 1]) && std.ascii.isHexDigit(src[i + 2]))
		{
			if (std.ascii.isLower(src[i + 1]) || std.ascii.isLower(src[i + 2]))
			{
				need_capitalize = true;
			}
			i += 2;
			// (b) needs escaping.
		}
		else if (!isASCII(src[i]))
		{
			num_to_escape++;
		}
		// (c) Already escaped and escape-characters normalized (eg. %2f -> %2F).
	}

	// Return if no changes needed.
	if (!num_to_escape && !need_capitalize) return src;

	auto ret = new char[num_to_escape * 2 + src.length];
	int j = 0;
	size_t i = 0;
	while (i < src.length)
	{
		// (a) Normalize %-escaped sequence (eg. %2f -> %2F).
		if (i + 2 < src.length && src[i] == '%'
				&& std.ascii.isHexDigit(src[i + 1]) && std.ascii.isHexDigit(src[i + 2]))
		{
			ret[j++] = src[i++];
			ret[j++] = std.ascii.toUpper(src[i++]);
			ret[j++] = std.ascii.toUpper(src[i]);
			// (b) %-escape octets whose highest bit is set. These are outside the
			// ASCII range.
		}
		else if (!isASCII(src[i]))
		{
			ret[j++] = '%';
			ret[j++] = kHexDigits[(src[i] >> 4) & 0xf];
			ret[j++] = kHexDigits[src[i] & 0xf];
			// (c) Normal character, no modification needed.
		}
		else
		{
			ret[j++] = src[i];
		}
		i++;
	}
	// ret is a locally allocated array
	return (() @trusted => ret.assumeUnique())();
}

unittest
{
	assert (MaybeEscapePattern("/SanJoséSellers") == "/SanJos%C3%A9Sellers");
	assert (MaybeEscapePattern("%aa") == "%AA");
}

/// Implements robots.txt pattern matching.
bool Matches(string path, string pattern) pure
{
	const pathlen = path.length;
	auto pos = new size_t[pathlen + 1];

	// The pos[] array holds a sorted list of indexes of 'path', with length
	// 'numpos'.  At the start and end of each iteration of the main loop below,
	// the pos[] array will hold a list of the prefixes of the 'path' which can
	// match the current prefix of 'pattern'. If this list is ever empty,
	// return false. If we reach the end of 'pattern' with at least one element
	// in pos[], return true.

	pos[0] = 0;
	size_t numpos = 1;

	foreach (idx, pat; pattern)
	{
		if (pat == '$' && idx == pattern.length - 1)
		{
			return pos[numpos - 1] == pathlen;
		}
		if (pat == '*')
		{
			numpos = pathlen - pos[0] + 1;
			foreach (i; 1 .. numpos)
			{
				pos[i] = pos[i - 1] + 1;
			}
		}
		else
		{
			// Includes '$' when not at end of pattern.
			size_t newnumpos = 0;
			foreach (i; 0 .. numpos)
			{
				if (pos[i] < pathlen && path[pos[i]] == pat)
				{
					pos[newnumpos++] = pos[i] + 1;
				}
			}
			numpos = newnumpos;
			if (numpos == 0) return false;
		}
	}

	return true;
}

package:

/// A robots.txt has lines of key/value pairs. A ParsedRobotsKey represents
/// a key. This class can parse a text-representation (including common typos)
/// and represent them as an enumeration which allows for faster processing
/// afterwards.
/// For unparsable keys, the original string representation is kept.
struct ParsedRobotsKey
{
public:

	enum KeyType
	{
		/// Generic highlevel fields.
		USER_AGENT,
		SITEMAP,

		/// Fields within a user-agent.
		ALLOW,
		DISALLOW,

		/// Unrecognized field; kept as-is. High number so that additions to the
		/// enumeration above does not change the serialization.
		UNKNOWN = 128
	}

	/// Parse given key text. Does not copy the text, so the text_key must stay
	/// valid for the object's life-time or the next Parse() call.
	void Parse(string key)
	{
		key_text_ = "";
		with (KeyType)
		{
			if (KeyIsUserAgent(key))
			{
				type_ = USER_AGENT;
			}
			else if (KeyIsAllow(key))
			{
				type_ = ALLOW;
			}
			else if (KeyIsDisallow(key))
			{
				type_ = DISALLOW;
			}
			else if (KeyIsSitemap(key))
			{
				type_ = SITEMAP;
			}
			else
			{
				type_ = UNKNOWN;
				key_text_ = key;
			}
		}
	}

	/// Returns the type of key.
	KeyType type() const
	{
		return type_;
	}

	/// If this is an unknown key, get the text.
	string GetUnknownText() const
	{
		assert (type_ == KeyType.UNKNOWN && !key_text_.empty);
		return key_text_;
	}

private:

	static bool KeyIsUserAgent(string key)
	{
		const ret = startsWithIgnoreCase(key, "user-agent");
		version (StrictSpelling)
		{
			return ret;
		}
		else
		{
			return ret || startsWithIgnoreCase(key, "useragent") || startsWithIgnoreCase(key, "user agent");
		}
	}

	static bool KeyIsAllow(string key)
	{
		return startsWithIgnoreCase(key, "allow");
	}

	static bool KeyIsDisallow(string key)
	{
		const ret = startsWithIgnoreCase(key, "disallow");
		version (StrictSpelling)
		{
			return ret;
		}
		else
		{
			return ret || startsWithIgnoreCase(key, "dissallow") || startsWithIgnoreCase(key, "dissalow") ||
				startsWithIgnoreCase(key, "disalow") || startsWithIgnoreCase(key, "diasllow") || startsWithIgnoreCase(key, "disallaw");
		}
	}

	static bool KeyIsSitemap(string key)
	{
		return startsWithIgnoreCase(key, "sitemap") || startsWithIgnoreCase(key, "site-map");
	}

	KeyType type_ = KeyType.UNKNOWN;
	string key_text_;
}

void emitKeyValueToHandler(int line, ref const(ParsedRobotsKey) key, string value,
		RobotsParseHandler handler)
{
	with (ParsedRobotsKey.KeyType) final switch (key.type)
	{
	case USER_AGENT:
		handler.HandleUserAgent(line, value);
		break;
	case ALLOW:
		handler.HandleAllow(line, value);
		break;
	case DISALLOW:
		handler.HandleDisallow(line, value);
		break;
	case SITEMAP:
		handler.HandleSitemap(line, value);
		break;
	case UNKNOWN:
		handler.HandleUnknownAction(line, key.GetUnknownText(), value);
		break;
	}
}

struct RobotsTxtParser
{
public:

	alias Key = ParsedRobotsKey;

	void Parse()
	{
		/// UTF-8 byte order marks.
		static immutable utf_bom = "\xef\xbb\xbf";

		/// Certain browsers limit the URL length to 2083 bytes. In a robots.txt, it's
		/// fairly safe to assume any valid line isn't going to be more than many times
		/// that max url length of 2KB. We want some padding for
		/// UTF-8 encoding/nulls/etc. but a much smaller bound would be okay as well.
		/// If so, we can ignore the chars on a line past that.
		enum kMaxLineLen = 2083 * 8 - 1;
		/// Allocate a buffer used to process the current line.
		auto line_buffer = new char[kMaxLineLen];
		size_t line_pos = 0;
		int line_num = 0;
		size_t bom_pos = 0;
		bool last_was_carriage_return = false;
		handler_.HandleRobotsStart();

		foreach (ch; robots_body_)
		{
			assert (line_pos <= kMaxLineLen);
			// Google-specific optimization: UTF-8 byte order marks should never
			// appear in a robots.txt file, but they do nevertheless. Skipping
			// possible BOM-prefix in the first bytes of the input.
			if (bom_pos < utf_bom.length && ch == utf_bom[bom_pos++]) continue;
			bom_pos = utf_bom.length;
			if (ch != '\x0a' && ch != '\x0d')
			{
				// Non-line-ending char case.
				// Put in next spot on current line, as long as there's room.
				if (line_pos < kMaxLineLen) line_buffer[line_pos++] = ch;
			}
			else
			{
				// Line-ending character char case.
				// Only emit an empty line if this was not due to the second character
				// of the DOS line-ending \r\n .
				const bool is_CRLF_continuation =
					line_pos == 0 && last_was_carriage_return && ch == '\x0a';
				if (!is_CRLF_continuation) ParseAndEmitLine(++line_num, line_buffer[0 .. line_pos]);
				line_pos = 0;
				last_was_carriage_return = (ch == '\x0d');
			}
		}
		ParseAndEmitLine(++line_num, line_buffer[0 .. line_pos]);
		handler_.HandleRobotsEnd();
	}

private:

	void ParseAndEmitLine(int current_line, char[] line)
	{
		string string_key, value;
		if (!getKeyAndValueFrom(string_key, value, line)) return;

		Key key;
		key.Parse(string_key);
		if (NeedEscapeValueForKey(key)) value = MaybeEscapePattern(value);
		emitKeyValueToHandler(current_line, key, value, handler_);
	}

	bool NeedEscapeValueForKey(ref const(Key) key)
	{
		with (Key.KeyType) return !key.type.among(USER_AGENT, SITEMAP);
	}

	string robots_body_;
	Rebindable!RobotsParseHandler handler_;
}

final class LongestMatchRobotsMatchStrategy : RobotsMatchStrategy
{
public:

	override int MatchAllow(string path, string pattern)
	{
		return Matches(path, pattern) ? pattern.length.to!int : -1;
	}

	override int MatchDisallow(string path, string pattern)
	{
		return Matches(path, pattern) ? pattern.length.to!int : -1;
	}
}

bool equalsIgnoreAsciiCase(string s1, string s2) @nogc pure
{
	return equal!((c1, c2) => std.ascii.toLower(c1) == std.ascii.toLower(c2))(s1.byChar, s2.byChar);
}

bool startsWithIgnoreCase(string target, string prefix) @nogc pure
{
	return target.length >= prefix.length && equalsIgnoreAsciiCase(target[0 .. prefix.length], prefix);
}

bool getKeyAndValueFrom(ref string key, ref string value, char[] line) pure
{
	auto line_c = line.byCodeUnit;
	static void asciiStripLeft(ref typeof(line_c) s) pure
	{
		s.findSkip!(std.ascii.isWhite);
	}

	static void asciiStripRight(ref typeof(line_c) s) pure
	{
		if (s.empty) return;
		const white_suffix_len = s.retro.countUntil!(c => !std.ascii.isWhite(c));
		s = s[0 .. $ - white_suffix_len];
	}
	// Remove comments from the current robots.txt line.
	if (auto comment_pieces = line_c.findSplitBefore("#"))
	{
		line_c = comment_pieces[0];
	}
	if (line_c.empty) return false;
	asciiStripLeft(line_c);
	asciiStripRight(line_c);

	// Rules must match the following pattern:
	//   <key>[ \t]*:[ \t]*<value>
	if (auto pieces = line_c.findSplit(":"))
	{
		asciiStripRight(pieces[0]);
		key = pieces[0].source.idup;
		asciiStripLeft(pieces[2]);
		value = pieces[2].source.idup;
	}
	else
	{
		// Google-specific optimization: some people forget the colon, so we need to
		// accept whitespace in its stead.
		auto after_whitespace = line_c.findAmong(" \t");
		if (after_whitespace.empty) return false;
		key = line_c[0 .. $ - after_whitespace.length].source.idup;
		asciiStripLeft(after_whitespace);
		// We only accept whitespace as a separator if there are exactly two
		// sequences of non-whitespace characters.  If we get here, there were
		// more than 2 such sequences since we stripped trailing whitespace
		// above.
		if (!after_whitespace.findAmong(" \t").empty) return false;
		value = after_whitespace.source.idup;
	}
	return !key.empty;
}

/// Extract the matchable part of a user agent string, essentially stopping at
/// the first invalid character.
/// Example: 'Googlebot/2.1' becomes 'Googlebot'
static string extractUserAgent(string user_agent) @nogc pure
{
	static bool isUserAgentChar(char c) pure @nogc
	{
		// Allowed characters in user-agent are [a-zA-Z_-].
		return std.ascii.isAlpha(c) || c == '-' || c == '_';
	}

	auto len = user_agent.byCodeUnit.countUntil!(c => !isUserAgentChar(c));
	if (len < 0) len = user_agent.length;
	return user_agent[0 .. len];
}

unittest
{
	assert (extractUserAgent("Googlebot/2.1") == "Googlebot");
	assert (extractUserAgent("Googlebot/2.1/2.1") == "Googlebot");
	assert (extractUserAgent("Googlebot2.1") == "Googlebot");
	assert (extractUserAgent("Googlebot") == "Googlebot");
	assert (extractUserAgent("/") == "");
	assert (extractUserAgent("") == "");
}