// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This file tests the robots.txt parsing and matching code found in robots.cc
// against the current Robots Exclusion Protocol (REP) internet draft (I-D).
// https://tools.ietf.org/html/draft-koster-rep

module unrobotstxt.test;

import unrobotstxt;

version (unittest)
{
	bool IsUserAgentAllowed(string robotstxt, string useragent, string url)
	{
		auto matcher = new RobotsMatcher;
		return matcher.OneAgentAllowedByRobots(robotstxt, useragent, url);
	}
}

// Google-specific: system test.
unittest
{
	// GoogleOnly_SystemTest
	immutable robotstxt =
		"user-agent: FooBot\n" ~
		"disallow: /\n";
	// Empty robots.txt: everything allowed.
	assert (IsUserAgentAllowed("", "FooBot", ""));

	// Empty user-agent to be matched: everything allowed.
	assert (IsUserAgentAllowed(robotstxt, "", ""));

	// Empty url: implicitly disallowed, see method comment for GetPathParamsQuery
	// in robots.cc.
	assert (!IsUserAgentAllowed(robotstxt, "FooBot", ""));

	// All params empty: same as robots.txt empty, everything allowed.
	assert (IsUserAgentAllowed("", "", ""));
}
// Rules are colon separated name-value pairs. The following names are
// provisioned:
//     user-agent: <value>
//     allow: <value>
//     disallow: <value>
// See REP I-D section "Protocol Definition".
// https://tools.ietf.org/html/draft-koster-rep#section-2.1
//
// Google specific: webmasters sometimes miss the colon separator, but it's
// obvious what they mean by "disallow /", so we assume the colon if it's
// missing.
unittest
{
	// ID_LineSyntax_Line
	immutable robotstxt_correct =
		"user-agent: FooBot\n" ~
		"disallow: /\n";
	immutable robotstxt_incorrect =
		"foo: FooBot\n" ~
		"bar: /\n";
	immutable robotstxt_incorrect_accepted =
		"user-agent FooBot\n" ~
		"disallow /\n";
	immutable url = "http://foo.bar/x/y";

	assert (!IsUserAgentAllowed(robotstxt_correct, "FooBot", url));
	assert (IsUserAgentAllowed(robotstxt_incorrect, "FooBot", url));
	assert (!IsUserAgentAllowed(robotstxt_incorrect_accepted, "FooBot", url));
}

// A group is one or more user-agent line followed by rules, and terminated
// by a another user-agent line. Rules for same user-agents are combined
// opaquely into one group. Rules outside groups are ignored.
// See REP I-D section "Protocol Definition".
// https://tools.ietf.org/html/draft-koster-rep#section-2.1
unittest
{
	// ID_LineSyntax_Groups
	immutable robotstxt =
		"allow: /foo/bar/\n" ~
		"\n" ~
		"user-agent: FooBot\n" ~
		"disallow: /\n" ~
		"allow: /x/\n" ~
		"user-agent: BarBot\n" ~
		"disallow: /\n" ~
		"allow: /y/\n" ~
		"\n" ~
		"\n" ~
		"allow: /w/\n" ~
		"user-agent: BazBot\n" ~
		"\n" ~
		"user-agent: FooBot\n" ~
		"allow: /z/\n" ~
		"disallow: /\n";

	immutable url_w = "http://foo.bar/w/a";
	immutable url_x = "http://foo.bar/x/b";
	immutable url_y = "http://foo.bar/y/c";
	immutable url_z = "http://foo.bar/z/d";
	immutable url_foo = "http://foo.bar/foo/bar/";

	assert (IsUserAgentAllowed(robotstxt, "FooBot", url_x));
	assert (IsUserAgentAllowed(robotstxt, "FooBot", url_z));
	assert (!IsUserAgentAllowed(robotstxt, "FooBot", url_y));
	assert (IsUserAgentAllowed(robotstxt, "BarBot", url_y));
	assert (IsUserAgentAllowed(robotstxt, "BarBot", url_w));
	assert (!IsUserAgentAllowed(robotstxt, "BarBot", url_z));
	assert (IsUserAgentAllowed(robotstxt, "BazBot", url_z));

	// Lines with rules outside groups are ignored.
	assert (!IsUserAgentAllowed(robotstxt, "FooBot", url_foo));
	assert (!IsUserAgentAllowed(robotstxt, "BarBot", url_foo));
	assert (!IsUserAgentAllowed(robotstxt, "BazBot", url_foo));
}

// REP lines are case insensitive. See REP I-D section "Protocol Definition".
// https://tools.ietf.org/html/draft-koster-rep#section-2.1
unittest
{
	// ID_REPLineNamesCaseInsensitive
	immutable robotstxt_upper =
		"USER-AGENT: FooBot\n" ~
		"ALLOW: /x/\n" ~
		"DISALLOW: /\n";
	immutable robotstxt_lower =
		"user-agent: FooBot\n" ~
		"allow: /x/\n" ~
		"disallow: /\n";
	immutable robotstxt_camel =
		"uSeR-aGeNt: FooBot\n" ~
		"AlLoW: /x/\n" ~
		"dIsAlLoW: /\n";
	immutable url_allowed = "http://foo.bar/x/y";
	immutable url_disallowed = "http://foo.bar/a/b";

	assert (IsUserAgentAllowed(robotstxt_upper, "FooBot", url_allowed));
	assert (IsUserAgentAllowed(robotstxt_lower, "FooBot", url_allowed));
	assert (IsUserAgentAllowed(robotstxt_camel, "FooBot", url_allowed));
	assert (!IsUserAgentAllowed(robotstxt_upper, "FooBot", url_disallowed));
	assert (!IsUserAgentAllowed(robotstxt_lower, "FooBot", url_disallowed));
	assert (!IsUserAgentAllowed(robotstxt_camel, "FooBot", url_disallowed));
}

// A user-agent line is expected to contain only [a-zA-Z_-] characters and must
// not be empty. See REP I-D section "The user-agent line".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.1
unittest
{
	// ID_VerifyValidUserAgentsToObey
	assert (RobotsMatcher.IsValidUserAgentToObey("Foobot"));
	assert (RobotsMatcher.IsValidUserAgentToObey("Foobot-Bar"));
	assert (RobotsMatcher.IsValidUserAgentToObey("Foo_Bar"));

	assert (!RobotsMatcher.IsValidUserAgentToObey(""));
	assert (!RobotsMatcher.IsValidUserAgentToObey("ツ"));

	assert (!RobotsMatcher.IsValidUserAgentToObey("Foobot*"));
	assert (!RobotsMatcher.IsValidUserAgentToObey(" Foobot "));
	assert (!RobotsMatcher.IsValidUserAgentToObey("Foobot/2.1"));

	assert (!RobotsMatcher.IsValidUserAgentToObey("Foobot Bar"));
}

// User-agent line values are case insensitive. See REP I-D section "The
// user-agent line".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.1
unittest
{
	// ID_UserAgentValueCaseInsensitive
	immutable robotstxt_upper =
		"User-Agent: FOO BAR\n" ~
		"Allow: /x/\n" ~
		"Disallow: /\n";
	immutable robotstxt_lower =
		"User-Agent: foo bar\n" ~
		"Allow: /x/\n" ~
		"Disallow: /\n";
	immutable robotstxt_camel =
		"User-Agent: FoO bAr\n" ~
		"Allow: /x/\n" ~
		"Disallow: /\n";
	immutable url_allowed = "http://foo.bar/x/y";
	immutable url_disallowed = "http://foo.bar/a/b";

	assert (IsUserAgentAllowed(robotstxt_upper, "Foo", url_allowed));
	assert (IsUserAgentAllowed(robotstxt_lower, "Foo", url_allowed));
	assert (IsUserAgentAllowed(robotstxt_camel, "Foo", url_allowed));
	assert (!IsUserAgentAllowed(robotstxt_upper, "Foo", url_disallowed));
	assert (!IsUserAgentAllowed(robotstxt_lower, "Foo", url_disallowed));
	assert (!IsUserAgentAllowed(robotstxt_camel, "Foo", url_disallowed));
	assert (IsUserAgentAllowed(robotstxt_upper, "foo", url_allowed));
	assert (IsUserAgentAllowed(robotstxt_lower, "foo", url_allowed));
	assert (IsUserAgentAllowed(robotstxt_camel, "foo", url_allowed));
	assert (!IsUserAgentAllowed(robotstxt_upper, "foo", url_disallowed));
	assert (!IsUserAgentAllowed(robotstxt_lower, "foo", url_disallowed));
	assert (!IsUserAgentAllowed(robotstxt_camel, "foo", url_disallowed));
}

// Google specific: accept user-agent value up to the first space. Space is not
// allowed in user-agent values, but that doesn't stop webmasters from using
// them. This is more restrictive than the I-D, since in case of the bad value
// "Googlebot Images" we'd still obey the rules with "Googlebot".
// Extends REP I-D section "The user-agent line"
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.1
unittest
{
	// GoogleOnly_AcceptUserAgentUpToFirstSpace
	assert (!RobotsMatcher.IsValidUserAgentToObey("Foobot Bar"));
	immutable robotstxt =
		"User-Agent: *\n" ~
		"Disallow: /\n" ~
		"User-Agent: Foo Bar\n" ~
		"Allow: /x/\n" ~
		"Disallow: /\n";
	immutable url = "http://foo.bar/x/y";

	assert (IsUserAgentAllowed(robotstxt, "Foo", url));
	assert (!IsUserAgentAllowed(robotstxt, "Foo Bar", url));
}

// If no group matches the user-agent, crawlers must obey the first group with a
// user-agent line with a "*" value, if present. If no group satisfies either
// condition, or no groups are present at all, no rules apply.
// See REP I-D section "The user-agent line".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.1
unittest
{
	// ID_GlobalGroups_Secondary
	immutable robotstxt_empty = "";
	immutable robotstxt_global =
		"user-agent: *\n" ~
		"allow: /\n" ~
		"user-agent: FooBot\n" ~
		"disallow: /\n";
	immutable robotstxt_only_specific =
		"user-agent: FooBot\n" ~
		"allow: /\n" ~
		"user-agent: BarBot\n" ~
		"disallow: /\n" ~
		"user-agent: BazBot\n" ~
		"disallow: /\n";
	immutable url = "http://foo.bar/x/y";

	assert (IsUserAgentAllowed(robotstxt_empty, "FooBot", url));
	assert (!IsUserAgentAllowed(robotstxt_global, "FooBot", url));
	assert (IsUserAgentAllowed(robotstxt_global, "BarBot", url));
	assert (IsUserAgentAllowed(robotstxt_only_specific, "QuxBot", url));
}

// Matching rules against URIs is case sensitive.
// See REP I-D section "The Allow and Disallow lines".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.2
unittest
{
	// ID_AllowDisallow_Value_CaseSensitive
	immutable robotstxt_lowercase_url =
		"user-agent: FooBot\n" ~
		"disallow: /x/\n";
	immutable robotstxt_uppercase_url =
		"user-agent: FooBot\n" ~
		"disallow: /X/\n";
	immutable url = "http://foo.bar/x/y";

	assert (!IsUserAgentAllowed(robotstxt_lowercase_url, "FooBot", url));
	assert (IsUserAgentAllowed(robotstxt_uppercase_url, "FooBot", url));
}

// The most specific match found MUST be used. The most specific match is the
// match that has the most octets. In case of multiple rules with the same
// length, the least strict rule must be used.
// See REP I-D section "The Allow and Disallow lines".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.2
unittest
{
	// ID_LongestMatch
	immutable url = "http://foo.bar/x/page.html";
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /x/page.html\n" ~
			"allow: /x/\n";

		assert (!IsUserAgentAllowed(robotstxt, "FooBot", url));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"allow: /x/page.html\n" ~
			"disallow: /x/\n";

		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/x/"));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: \n" ~
			"allow: \n";
		// In case of equivalent disallow and allow patterns for the same
		// user-agent, allow is used.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n" ~
			"allow: /\n";
		// In case of equivalent disallow and allow patterns for the same
		// user-agent, allow is used.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
	}
	{
		immutable url_a = "http://foo.bar/x";
		immutable url_b = "http://foo.bar/x/";
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /x\n" ~
			"allow: /x/\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", url_a));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url_b));
	}

	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /x/page.html\n" ~
			"allow: /x/page.html\n";
		// In case of equivalent disallow and allow patterns for the same
		// user-agent, allow is used.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"allow: /page\n" ~
			"disallow: /*.html\n";
		// Longest match wins.
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/page.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/page"));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"allow: /x/page.\n" ~
			"disallow: /*.html\n";
		// Longest match wins.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/x/y.html"));
	}
	{
		immutable robotstxt =
			"User-agent: *\n" ~
			"Disallow: /x/\n" ~
			"User-agent: FooBot\n" ~
			"Disallow: /y/\n";
		// Most specific group for FooBot allows implicitly /x/page.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/x/page"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/y/page"));
	}
}

// Octets in the URI and robots.txt paths outside the range of the US-ASCII
// coded character set, and those in the reserved range defined by RFC3986,
// MUST be percent-encoded as defined by RFC3986 prior to comparison.
// See REP I-D section "The Allow and Disallow lines".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.2
//
// NOTE: It's up to the caller to percent encode a URL before passing it to the
// parser. Percent encoding URIs in the rules is unnecessary.
unittest
{
	// ID_Encoding
	// /foo/bar?baz=http://foo.bar stays unencoded.
	{
		immutable robotstxt =
			"User-agent: FooBot\n" ~
			"Disallow: /\n" ~
			"Allow: /foo/bar?qux=taz&baz=http://foo.bar?tar&par\n";
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar?qux=taz&baz=http://foo.bar?tar&par"));
	}

	// 3 byte character: /foo/bar/ツ -> /foo/bar/%E3%83%84
	{
		immutable robotstxt =
			"User-agent: FooBot\n" ~
			"Disallow: /\n" ~
			"Allow: /foo/bar/ツ\n";
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/%E3%83%84"));
		// The parser encodes the 3-byte character, but the URL is not %-encoded.
		assert (!
				IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/ツ"));
	}
	// Percent encoded 3 byte character: /foo/bar/%E3%83%84 -> /foo/bar/%E3%83%84
	{
		immutable robotstxt =
			"User-agent: FooBot\n" ~
			"Disallow: /\n" ~
			"Allow: /foo/bar/%E3%83%84\n";
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/%E3%83%84"));
		assert (!
				IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/ツ"));
	}
	// Percent encoded unreserved US-ASCII: /foo/bar/%62%61%7A -> NULL
	// This is illegal according to RFC3986 and while it may work here due to
	// simple string matching, it should not be relied on.
	{
		immutable robotstxt =
			"User-agent: FooBot\n" ~
			"Disallow: /\n" ~
			"Allow: /foo/bar/%62%61%7A\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/baz"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/%62%61%7A"));
	}
}

// The REP I-D defines the following characters that have special meaning in
// robots.txt:
// # - inline comment.
// $ - end of pattern.
// * - any number of characters.
// See REP I-D section "Special Characters".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.3
unittest
{
	// ID_SpecialCharacters
	{
		immutable robotstxt =
			"User-agent: FooBot\n" ~
			"Disallow: /foo/bar/quz\n" ~
			"Allow: /foo/*/qux\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/quz"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/quz"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo//quz"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bax/quz"));
	}
	{
		immutable robotstxt =
			"User-agent: FooBot\n" ~
			"Disallow: /foo/bar$\n" ~
			"Allow: /foo/bar/qux\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/qux"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar/baz"));
	}
	{
		immutable robotstxt =
			"User-agent: FooBot\n" ~
			"# Disallow: /\n" ~
			"Disallow: /foo/quz#qux\n" ~
			"Allow: /\n";
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/bar"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/foo/quz"));
	}
}

// Google-specific: "index.html" (and only that) at the end of a pattern is
// equivalent to "/".
unittest
{
	// GoogleOnly_IndexHTMLisDirectory
	immutable robotstxt =
		"User-Agent: *\n" ~
		"Allow: /allowed-slash/index.html\n" ~
		"Disallow: /\n";
	// If index.html is allowed, we interpret this as / being allowed too.
	assert (IsUserAgentAllowed(robotstxt, "foobot", "http://foo.com/allowed-slash/"));
	// Does not exatly match.
	assert (!IsUserAgentAllowed(robotstxt, "foobot", "http://foo.com/allowed-slash/index.htm"));
	// Exact match.
	assert (IsUserAgentAllowed(robotstxt, "foobot", "http://foo.com/allowed-slash/index.html"));
	assert (!IsUserAgentAllowed(robotstxt, "foobot", "http://foo.com/anyother-url"));
}

// Google-specific: long lines are ignored after 8 * 2083 bytes. See comment in
// RobotsTxtParser::Parse().
unittest
{
	// GoogleOnly_LineTooLong
	immutable kEOLLen = "\n".length;
	immutable size_t kMaxLineLen = 2083 * 8;
	immutable allow = "allow: ";
	immutable disallow = "disallow: ";

	// Disallow rule pattern matches the URL after being cut off at kMaxLineLen.
	{
		string robotstxt = "user-agent: FooBot\n";
		string longline = "/x/";
		immutable size_t max_length = kMaxLineLen - longline.length - disallow.length + kEOLLen;
		while (longline.length < max_length)
		{
			longline ~= "a";
		}
		robotstxt ~= disallow ~ longline ~ "/qux\n";

		// Matches nothing, so URL is allowed.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fux"));
		// Matches cut off disallow rule.
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar" ~ longline ~ "/fux"));
	}

	{
		string robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n";
		string longline_a = "/x/";
		string longline_b = "/x/";
		immutable size_t max_length = kMaxLineLen - longline_a.length - allow.length + kEOLLen;
		while (longline_a.length < max_length)
		{
			longline_a ~= "a";
			longline_b ~= "b";
		}
		robotstxt ~= allow ~ longline_a ~ "/qux\n";
		robotstxt ~= allow ~ longline_b ~ "/qux\n";

		// URL matches the disallow rule.
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/"));
		// Matches the allow rule exactly.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar" ~ longline_a ~ "/qux"));
		// Matches cut off allow rule.
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar" ~ longline_b ~ "/fux"));
	}
}

unittest
{
	// GoogleOnly_DocumentationChecks
	// Test documentation from
	// https://developers.google.com/search/reference/robots_txt
	// Section "URL matching based on path values".
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n" ~
			"allow: /fish\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/bar"));

		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish/salmon.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fishheads"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fishheads/yummy.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish.html?id=anything"));

		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/Fish.asp"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/catfish"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/?id=fish"));
	}
	// "/fish*" equals "/fish"
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n" ~
			"allow: /fish*\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/bar"));

		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish/salmon.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fishheads"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fishheads/yummy.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish.html?id=anything"));

		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/Fish.bar"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/catfish"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/?id=fish"));
	}
	// "/fish/" does not equal "/fish"
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n" ~
			"allow: /fish/\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/bar"));

		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish/"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish/salmon"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish/?salmon"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish/salmon.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish/?id=anything"));

		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish.html"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/Fish/Salmon.html"));
	}
	// "/*.php"
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n" ~
			"allow: /*.php\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/bar"));

		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/filename.php"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/folder/filename.php"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/folder/filename.php?parameters"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar//folder/any.php.file.html"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/filename.php/"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/index?f=filename.php/"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/php/"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/index?php"));

		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/windows.PHP"));
	}
	// "/*.php$"
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n" ~
			"allow: /*.php$\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/bar"));

		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/filename.php"));
		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/folder/filename.php"));

		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/filename.php?parameters"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/filename.php/"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/filename.php5"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/php/"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/filename?php"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/aaaphpaaa"));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar//windows.PHP"));
	}
	// "/fish*.php"
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"disallow: /\n" ~
			"allow: /fish*.php\n";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/bar"));

		assert (IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fish.php"));
		assert ( IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/fishheads/catfish.php?parameters"));

		assert (!IsUserAgentAllowed(robotstxt, "FooBot", "http://foo.bar/Fish.PHP"));
	}
	// Section "Order of precedence for group-member records".
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"allow: /p\n" ~
			"disallow: /\n";
		immutable url = "http://example.com/page";
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"allow: /folder\n" ~
			"disallow: /folder\n";
		immutable url = "http://example.com/folder/page";
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"allow: /page\n" ~
			"disallow: /*.htm\n";
		immutable url = "http://example.com/page.htm";
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", url));
	}
	{
		immutable robotstxt =
			"user-agent: FooBot\n" ~
			"allow: /$\n" ~
			"disallow: /\n";
		immutable url = "http://example.com/";
		immutable url_page = "http://example.com/page.html";
		assert (IsUserAgentAllowed(robotstxt, "FooBot", url));
		assert (!IsUserAgentAllowed(robotstxt, "FooBot", url_page));
	}
}

version (unittest)
{
	class RobotsStatsReporter : RobotsParseHandler
	{
		public:
			override void HandleRobotsStart() 
			{
				last_line_seen_ = 0;
				valid_directives_ = 0;
				unknown_directives_ = 0;
				sitemap_ = "";
			}
			override void HandleRobotsEnd() {}

			override void HandleUserAgent(int line_num, string value) 
			{
				Digest(line_num);
			}
			override void HandleAllow(int line_num, string value) 
			{
				Digest(line_num);
			}
			override void HandleDisallow(int line_num, string value) 
			{
				Digest(line_num);
			}

			override void HandleSitemap(int line_num, string value) 
			{
				Digest(line_num);
				sitemap_ ~= value;
			}

			// Any other unrecognized name/v pairs.
			override void HandleUnknownAction(int line_num, string action, string value) 
			{
				last_line_seen_ = line_num;
				unknown_directives_++;
			}

			int last_line_seen() const { return last_line_seen_; }

			// All directives found, including unknown.
			int valid_directives() const { return valid_directives_; }

			// Number of unknown directives.
			int unknown_directives() const { return unknown_directives_; }

			// Parsed sitemap line.
			string sitemap() const { return sitemap_; }

		private:
			void Digest(int line_num) @safe
			{
				assert (line_num >= last_line_seen_);
				last_line_seen_ = line_num;
				valid_directives_++;
			}

			int last_line_seen_ = 0;
			int valid_directives_ = 0;
			int unknown_directives_ = 0;
			string sitemap_;
	}
}

// Different kinds of line endings are all supported: %x0D / %x0A / %x0D.0A
unittest
{
	// ID_LinesNumbersAreCountedCorrectly

	auto report = new RobotsStatsReporter;

	immutable string kUnixFile =
		"User-Agent: foo\n" ~
		"Allow: /some/path\n" ~
		"User-Agent: bar\n" ~
		"\n" ~
		"\n" ~
		"Disallow: /\n";
	ParseRobotsTxt(kUnixFile, report);
	assert (report.valid_directives() == 4);
	assert (report.last_line_seen() == 6);

	immutable string kDosFile =
		"User-Agent: foo\r\n" ~
		"Allow: /some/path\r\n" ~
		"User-Agent: bar\r\n" ~
		"\r\n" ~
		"\r\n" ~
		"Disallow: /\r\n";
	ParseRobotsTxt(kDosFile, report);
	assert (report.valid_directives() == 4);
	assert (report.last_line_seen() == 6);

	immutable string kMacFile =
		"User-Agent: foo\r" ~
		"Allow: /some/path\r" ~
		"User-Agent: bar\r" ~
		"\r" ~
		"\r" ~
		"Disallow: /\r";
	ParseRobotsTxt(kMacFile, report);
	assert (report.valid_directives() == 4);
	assert (report.last_line_seen() == 6);

	immutable string kNoFinalNewline =
		"User-Agent: foo\n" ~
		"Allow: /some/path\n" ~
		"User-Agent: bar\n" ~
		"\n" ~
		"\n" ~
		"Disallow: /";
	ParseRobotsTxt(kNoFinalNewline, report);
	assert (report.valid_directives() == 4);
	assert (report.last_line_seen() == 6);

	immutable string kMixedFile =
		"User-Agent: foo\n" ~
		"Allow: /some/path\r\n" ~
		"User-Agent: bar\n" ~
		"\r\n" ~
		"\n" ~
		"Disallow: /";
	ParseRobotsTxt(kMixedFile, report);
	assert (report.valid_directives() == 4);
	assert (report.last_line_seen() == 6);
}

// BOM characters are unparseable and thus skipped. The rules following the line
// are used.
unittest
{
	// ID_UTF8ByteOrderMarkIsSkipped

	auto report = new RobotsStatsReporter;

	immutable kUtf8FileFullBOM =
		"\xEF\xBB\xBF" ~
		"User-Agent: foo\n" ~
		"Allow: /AnyValue\n";
	ParseRobotsTxt(kUtf8FileFullBOM, report);
	assert (report.valid_directives() == 2);
	assert (report.unknown_directives() == 0);

	// We allow as well partial ByteOrderMarks.
	immutable kUtf8FilePartial2BOM =
		"\xEF\xBB" ~
		"User-Agent: foo\n" ~
		"Allow: /AnyValue\n";
	ParseRobotsTxt(kUtf8FilePartial2BOM, report);
	assert (report.valid_directives() == 2);
	assert (report.unknown_directives() == 0);

	immutable kUtf8FilePartial1BOM =
		"\xEF" ~
		"User-Agent: foo\n" ~
		"Allow: /AnyValue\n";
	ParseRobotsTxt(kUtf8FilePartial1BOM, report);
	assert (report.valid_directives() == 2);
	assert (report.unknown_directives() == 0);

	// If the BOM is not the right sequence, the first line looks like garbage
	// that is skipped (we essentially see "\x11\xBFUser-Agent").
	immutable kUtf8FileBrokenBOM =
		"\xEF\x11\xBF" ~
		"User-Agent: foo\n" ~
		"Allow: /AnyValue\n";
	ParseRobotsTxt(kUtf8FileBrokenBOM, report);
	assert (report.valid_directives() == 1);
	assert (report.unknown_directives() == 1);  // We get one broken line.

	// Some other messed up file: BOMs only valid in the beginning of the file.
	immutable kUtf8BOMSomewhereInMiddleOfFile =
		"User-Agent: foo\n" ~
		"\xEF\xBB\xBF" ~
		"Allow: /AnyValue\n";
	ParseRobotsTxt(kUtf8BOMSomewhereInMiddleOfFile, report);
	assert (report.valid_directives() == 1);
	assert (report.unknown_directives() == 1);
}

// Google specific: the I-D allows any line that crawlers might need, such as
// sitemaps, which Google supports.
// See REP I-D section "Other records".
// https://tools.ietf.org/html/draft-koster-rep#section-2.2.4
unittest
{
	// ID_NonStandardLineExample_Sitemap
	auto report = new RobotsStatsReporter;

	{
		auto sitemap_loc = "http://foo.bar/sitemap.xml";
		auto robotstxt =
			"User-Agent: foo\n" ~
			"Allow: /some/path\n" ~
			"User-Agent: bar\n" ~
			"\n" ~
			"\n" ~
			"Sitemap: " ~ sitemap_loc ~ "\n";
		ParseRobotsTxt(robotstxt, report);
		assert (sitemap_loc == report.sitemap);
	}

	// A sitemap line may appear anywhere in the file.
	{
		auto sitemap_loc = "http://foo.bar/sitemap.xml";
		auto robotstxt =
			"Sitemap: " ~ sitemap_loc ~ "\n" ~
			"User-Agent: foo\n" ~
			"Allow: /some/path\n" ~
			"User-Agent: bar\n" ~
			"\n" ~
			"\n";
		ParseRobotsTxt(robotstxt, report);
		assert (sitemap_loc == report.sitemap);
	}
}

version (unittest)
{
	void TestPath(string url, string expected_path)
	{
		assert (expected_path == GetPathParamsQuery(url));
	}

	void TestEscape(string url, string expected)
	{
		assert (expected == MaybeEscapePattern(url));
	}
}

unittest
{
	// TestGetPathParamsQuery
	// Only testing URLs that are already correctly escaped here.
	TestPath("", "/");
	TestPath("http://www.example.com", "/");
	TestPath("http://www.example.com/", "/");
	TestPath("http://www.example.com/a", "/a");
	TestPath("http://www.example.com/a/", "/a/");
	TestPath("http://www.example.com/a/b?c=http://d.e/", "/a/b?c=http://d.e/");
	TestPath("http://www.example.com/a/b?c=d&e=f#fragment", "/a/b?c=d&e=f");
	TestPath("example.com", "/");
	TestPath("example.com/", "/");
	TestPath("example.com/a", "/a");
	TestPath("example.com/a/", "/a/");
	TestPath("example.com/a/b?c=d&e=f#fragment", "/a/b?c=d&e=f");
	TestPath("a", "/");
	TestPath("a/", "/");
	TestPath("/a", "/a");
	TestPath("a/b", "/b");
	TestPath("example.com?a", "/?a");
	TestPath("example.com/a;b#c", "/a;b");
	TestPath("//a/b/c", "/b/c");
}

unittest
{
	// TestMaybeEscapePattern
	TestEscape("http://www.example.com", "http://www.example.com");
	TestEscape("/a/b/c", "/a/b/c");
	TestEscape("á", "%C3%A1");
	TestEscape("%aa", "%AA");
}