Implementation of the robots.txt parser, matcher, and per-host cache. More...

#include "mdl_robots.h"
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "ra8_attributes.h"

Include dependency graph for mdl_robots.c:

Enumerations
enum	mdl_robots_local_size_t : uint16_t { k_robots_line_max = 512 , k_robots_url_max = 256 }
	Local parse/format sizes. More...
enum	mdl_robots_ms_t : uint32_t { k_ms_per_s = 1000U , k_crawl_cap_ms = 60000U }
	Millisecond conversions and the crawl-delay ceiling. More...
enum	mdl_robots_spec_t : int8_t { k_spec_none = -1 }
	"No user-agent match" sentinel for the specificity score. More...

Functions
static char	internal_lower_ascii (char c)
	ASCII lower-case of one character (locale-independent).
static bool	internal_ieq (const char a, const char b)
	Case-insensitive equality of two NUL-terminated strings.
static bool	internal_ci_prefix (const char prefix, const char s)
	True if prefix (case-insensitive) is a prefix of s.
static char *	internal_trim_ws (char *s)
	Trim leading/trailing ASCII whitespace in place; return start.
static bool	internal_next_line (const char *pp, const char end, char buf, size_t cap, bool truncated)
	Read one line into buf; strip its # comment and trim whitespace.
static bool	internal_split_field (char line, char field, char *value)
	Split a trimmed line into field/value on the first :.
static int	internal_agent_spec (const char agent, const char ua_token)
	Specificity of a robots user-agent token vs ours (-1 = no match).
static bool	internal_scan_spec (const char text, size_t len, const char ua_token, int best, bool wild)
	Pass 1: find the best specific match length and wildcard presence.
static bool	internal_group_selected (int target, bool wildcard_target, int grp_spec, bool grp_wild)
	True if a group of this specificity is the selected group.
static void	internal_add_rule (mdl_robots_t out, mdl_robots_rule_kind_t kind, const char value)
	Append an Allow/Disallow rule (non-empty patterns only).
static void	internal_set_crawl (mdl_robots_t out, const char value)
	Record the strictest Crawl-delay seen (clamped to the ceiling).
static void	internal_apply_directive (mdl_robots_t out, const char field, const char *value)
	Apply one non-user-agent directive line to the selected group.
static void	internal_harvest (const char text, size_t len, const char ua_token, int target, bool wildcard_target, mdl_robots_t *out)
	Pass 2: collect rules/crawl-delay from the selected group(s).
void	mdl_robots_parse (const char text, size_t len, const char ua_token, mdl_robots_t *out)
	Parse robots.txt text for the group matching ua_token.
static bool	internal_glob_prefix (const char pat, size_t patlen, const char path, bool anchored)
	Match pat (len patlen, * glob) against a prefix of path.
static bool	internal_robots_match (const char pat, size_t len, const char path)
	Match a rule pattern (honouring a trailing $) against path.
static const mdl_robots_rule_t *	internal_best_matching_rule (const mdl_robots_t robots, const char path)
	Longest matching rule for path (Allow wins ties), or NULL.
bool	mdl_robots_allows (const mdl_robots_t robots, const char path)
	Decide whether path is allowed by a parsed robots group.
const char *	mdl_robots_disallow_reason (const mdl_robots_t robots, const char path)
	Return the Disallow pattern that forbids path, or NULL if allowed.
static mdl_robots_cache_entry_t *	internal_cache_find (mdl_robots_cache_t cache, const char scheme, const char *host)
	Locate an existing cache entry for one origin, or NULL if absent.
static mdl_robots_cache_entry_t *	internal_cache_slot (mdl_robots_cache_t *cache)
	Pick a free cache slot, or the dedicated uncached overflow slot.
const mdl_robots_t *	mdl_robots_cache_consult (mdl_robots_cache_t cache, const char scheme, const char host, const char ua_token, mdl_robots_fetch_fn fetch, void ctx, char scratch, size_t scratch_cap)
	Return the cached robots rules for host, fetching on first contact.

Detailed Description

Implementation of the robots.txt parser, matcher, and per-host cache.

Parses a bounded robots document in two passes, selects the most specific applicable user-agent group, and retains its rules in fixed caller storage. Matching and cache lookup remain allocation-free and network access is supplied through the public fetch callback seam.

Copyright: Copyright (c) 2026 Brighton Sikarskie SPDX-License-Identifier: MIT

Definition in file mdl_robots.c.

Enumeration Type Documentation

◆ mdl_robots_local_size_t

enum mdl_robots_local_size_t : uint16_t

Local parse/format sizes.

Enumerator
k_robots_line_max	Max robots.txt line length.
k_robots_url_max	Max /robots.txt URL length.

Definition at line 22 of file mdl_robots.c.

◆ mdl_robots_ms_t

enum mdl_robots_ms_t : uint32_t

Millisecond conversions and the crawl-delay ceiling.

Enumerator
k_ms_per_s	Milliseconds per second.
k_crawl_cap_ms	Clamp a Crawl-delay to at most 60s.

Definition at line 28 of file mdl_robots.c.

◆ mdl_robots_spec_t

enum mdl_robots_spec_t : int8_t

"No user-agent match" sentinel for the specificity score.

Enumerator
k_spec_none	No group matched our user-agent.

Definition at line 34 of file mdl_robots.c.

Function Documentation

◆ internal_add_rule()

void internal_add_rule	(	mdl_robots_t *	out,
		mdl_robots_rule_kind_t	kind,
		const char *	value )

static

Append an Allow/Disallow rule (non-empty patterns only).

Empty patterns are no-ops; capacity or length overflow invalidates the parse.

Parameters

[in,out]	out	Parsed rule set.
[in]	kind	Allow or disallow classification.
[in]	value	NUL-terminated rule pattern.

Returns: Nothing.

Precondition: out and value are non-NULL.; out owns writable fixed rule storage.

Postcondition: A fitting non-empty rule increments the count once.; Overflow clears valid without writing past bounds.

Note: Not thread-safe: mutates out.

Since: 0.1.0

Definition at line 313 of file mdl_robots.c.

References mdl_robots_t::count, k_mdl_robots_max_rules, k_mdl_robots_path_max, mdl_robots_rule_t::kind, mdl_robots_rule_t::len, mdl_robots_rule_t::path, mdl_robots_t::rules, strlen(), strnlen(), and mdl_robots_t::valid.

Referenced by internal_apply_directive().

◆ internal_agent_spec()

int internal_agent_spec	(	const char *	agent,
		const char *	ua_token )

static

Specificity of a robots user-agent token vs ours (-1 = no match).

Wildcard scores zero; a specific ASCII-folded prefix scores its length.

Parameters

[in]	agent	Robots user-agent value.
[in]	ua_token	This tool's product token.

Returns: Match specificity.

Return values

-1	No match.
other	Zero for wildcard or a positive prefix length.

Precondition: Both arguments are non-NULL and NUL-terminated.; Robots product-token prefix matching is intended.

Postcondition: Inputs are unchanged.; No state is modified.

Note: Thread-safe: reads only arguments.

Since: 0.1.0

Definition at line 215 of file mdl_robots.c.

References internal_ci_prefix(), k_spec_none, RA8_INTERNAL, strcmp(), and strlen().

Referenced by internal_harvest(), and internal_scan_spec().

◆ internal_apply_directive()

void internal_apply_directive	(	mdl_robots_t *	out,
		const char *	field,
		const char *	value )

static

Apply one non-user-agent directive line to the selected group.

Dispatches Allow, Disallow, and Crawl-delay; unknown fields are ignored.

Parameters

[in,out]	out	Parsed robots result.
[in]	field	Normalised directive name.
[in]	value	Trimmed directive value.

Returns: Nothing.

Precondition: All arguments are non-NULL and NUL-terminated.; out owns writable fixed storage.

Postcondition: Recognised directives update only their policy field.; Unknown directives leave out unchanged.

Note: Not thread-safe: may mutate out.

Since: 0.1.0

Definition at line 376 of file mdl_robots.c.

References internal_add_rule(), internal_ieq(), internal_set_crawl(), k_mdl_rule_allow, and k_mdl_rule_disallow.

Referenced by internal_harvest().

◆ internal_best_matching_rule()

const mdl_robots_rule_t * internal_best_matching_rule	(	const mdl_robots_t *	robots,
		const char *	path )

static

Longest matching rule for path (Allow wins ties), or NULL.

Definition at line 559 of file mdl_robots.c.

References mdl_robots_t::count, internal_robots_match(), k_mdl_rule_allow, k_spec_none, mdl_robots_rule_t::kind, mdl_robots_rule_t::len, mdl_robots_rule_t::path, RA8_INTERNAL, and mdl_robots_t::rules.

Referenced by mdl_robots_allows(), and mdl_robots_disallow_reason().

◆ internal_cache_find()

mdl_robots_cache_entry_t * internal_cache_find	(	mdl_robots_cache_t *	cache,
		const char *	scheme,
		const char *	host )

static

Locate an existing cache entry for one origin, or NULL if absent.

Definition at line 598 of file mdl_robots.c.

References mdl_robots_cache_entry_t::host, mdl_robots_cache_t::hosts, k_mdl_robots_max_hosts, mdl_robots_cache_entry_t::scheme, strcmp(), and mdl_robots_cache_entry_t::used.

Referenced by mdl_robots_cache_consult().

◆ internal_cache_slot()

mdl_robots_cache_entry_t * internal_cache_slot ( mdl_robots_cache_t * cache )

static

Pick a free cache slot, or the dedicated uncached overflow slot.

Definition at line 610 of file mdl_robots.c.

References mdl_robots_cache_t::hosts, k_mdl_robots_max_hosts, mdl_robots_cache_t::overflow, RA8_INTERNAL, and mdl_robots_cache_entry_t::used.

Referenced by mdl_robots_cache_consult().

◆ internal_ci_prefix()

bool internal_ci_prefix	(	const char *	prefix,
		const char *	s )

static

True if prefix (case-insensitive) is a prefix of s.

Compares the candidate prefix using ASCII folding.

Parameters

[in]	prefix	Candidate prefix.
[in]	s	String to inspect.

Returns: Whether the folded prefix matches.

Return values

true	Every prefix byte matches.
false	A byte differs.

Precondition: Both strings are non-NULL and NUL-terminated.; s is readable through the candidate prefix length.

Postcondition: Inputs are unchanged.; No state is modified.

Note: Thread-safe: reads only arguments.

Since: 0.1.0

Definition at line 95 of file mdl_robots.c.

References internal_lower_ascii(), and RA8_INTERNAL.

Referenced by internal_agent_spec().

◆ internal_glob_prefix()

bool internal_glob_prefix	(	const char *	pat,
		size_t	patlen,
		const char *	path,
		bool	anchored )

static

Match pat (len patlen, * glob) against a prefix of path.

Uses bounded wildcard backtracking and optionally requires complete path consumption.

Parameters

[in]	pat	Pattern bytes.
[in]	patlen	Pattern length without a terminal anchor.
[in]	path	NUL-terminated URL path.
[in]	anchored	Whether the match must consume the complete path.

Returns: Whether the pattern matches.

Return values

true	The required prefix or full path matched.
false	No wildcard expansion matched.

Precondition: pat and path are non-NULL and readable for their lengths.; patlen excludes a terminal $ marker.

Postcondition: Inputs are unchanged.; No state is modified.

Note: Runtime is bounded by fixed rule and path limits.

Since: 0.1.0

Definition at line 499 of file mdl_robots.c.

Referenced by internal_robots_match().

◆ internal_group_selected()

bool internal_group_selected	(	int	target,
		bool	wildcard_target,
		int	grp_spec,
		bool	grp_wild )

static

True if a group of this specificity is the selected group.

Selects wildcard membership only for wildcard fallback, else exact specificity.

Parameters

[in]	target	Selected specific score.
[in]	wildcard_target	Whether wildcard fallback was selected.
[in]	grp_spec	Current group's score.
[in]	grp_wild	Whether the current group contains wildcard.

Returns: Whether the group contributes directives.

Return values

true	The group matches the selected target.
false	The group is not selected.

Precondition: Scores use the parser's sentinel convention.; Flags describe their corresponding groups.

Postcondition: No state is modified.; Inputs are unchanged.

Note: Thread-safe: pure comparison.

Since: 0.1.0

Definition at line 293 of file mdl_robots.c.

Referenced by internal_harvest().

◆ internal_harvest()

void internal_harvest	(	const char *	text,
		size_t	len,
		const char *	ua_token,
		int	target,
		bool	wildcard_target,
		mdl_robots_t *	out )

static

Pass 2: collect rules/crawl-delay from the selected group(s).

Rescans the document and applies directives only while a selected group is active.

Parameters

[in]	text	Robots document bytes.
[in]	len	Readable document length.
[in]	ua_token	Product token used for specificity.
[in]	target	Selected specific score.
[in]	wildcard_target	Whether wildcard fallback was selected.
[in,out]	out	Result receiving directives.

Returns: Nothing.

Precondition: Pointer arguments are non-NULL and text is readable for len bytes.; Selection inputs came from the first pass.

Postcondition: Only selected-group directives contribute to out.; Any truncated line invalidates out.

Note: Not thread-safe: mutates out.

Since: 0.1.0

Definition at line 404 of file mdl_robots.c.

References internal_agent_spec(), internal_apply_directive(), internal_group_selected(), internal_ieq(), internal_next_line(), internal_split_field(), k_robots_line_max, k_spec_none, RA8_INTERNAL, and mdl_robots_t::valid.

Referenced by mdl_robots_parse().

◆ internal_ieq()

bool internal_ieq	(	const char *	a,
		const char *	b )

static

Case-insensitive equality of two NUL-terminated strings.

Compares ASCII-folded bytes through both terminators.

Parameters

[in]	a	First string.
[in]	b	Second string.

Returns: Whether the strings are equal ignoring ASCII case.

Return values

true	All folded bytes match.
false	A byte or length differs.

Precondition: a and b are non-NULL and NUL-terminated.; ASCII-only folding is sufficient.

Postcondition: Inputs are unchanged.; No state is modified.

Note: Thread-safe: reads only arguments.

Since: 0.1.0

Definition at line 71 of file mdl_robots.c.

References internal_lower_ascii(), and RA8_INTERNAL.

Referenced by internal_apply_directive(), internal_harvest(), and internal_scan_spec().

◆ internal_lower_ascii()

char internal_lower_ascii ( char c )

static

ASCII lower-case of one character (locale-independent).

Maps A through Z; all other values pass through.

Parameters

[in] c Character to map.

Returns: Lower-case ASCII equivalent.

Return values

other Mapped or unchanged character.

Precondition: c is representable as char.; Locale-specific folding is not required.

Postcondition: No state is modified.; Non-uppercase input is unchanged.

Note: Thread-safe: pure arithmetic.

Since: 0.1.0

Definition at line 51 of file mdl_robots.c.

References RA8_INTERNAL.

Referenced by internal_ci_prefix(), and internal_ieq().

◆ internal_next_line()

bool internal_next_line	(	const char **	pp,
		const char *	end,
		char *	buf,
		size_t	cap,
		bool *	truncated )

static

Read one line into buf; strip its # comment and trim whitespace.

Consumes through the next newline or end. Bytes that exceed the buffer are still consumed and set the caller's sticky truncation flag, while the retained prefix is always NUL-terminated before comment processing.

Parameters

[in,out]	pp	Cursor updated to the start of the following line.
[in]	end	One-past-last byte of the robots document.
[out]	buf	Storage receiving the bounded line prefix.
[in]	cap	Capacity of `buf` in bytes.
[in,out]	truncated	Sticky flag set when a line does not fit.

Returns: Whether a line was available to consume.

Return values

true	One line was consumed and `buf` was populated.
false	The cursor was already at or beyond `end`.

Precondition: pp, buf, and truncated are non-NULL and cap is non-zero.; *pp and end delimit one readable contiguous byte range.

Postcondition: On true, *pp advances past the consumed line and optional newline.; On true, buf is NUL-terminated and contains no inline comment.

Note: The function preserves an already-true truncated value.

Since: 0.1.0

Definition at line 144 of file mdl_robots.c.

References strchr().

Referenced by internal_harvest(), and internal_scan_spec().

◆ internal_robots_match()

bool internal_robots_match	(	const char *	pat,
		size_t	len,
		const char *	path )

static

Match a rule pattern (honouring a trailing $) against path.

Removes a terminal anchor from the glob length and delegates matching.

Parameters

[in]	pat	Rule pattern bytes.
[in]	len	Stored pattern length.
[in]	path	NUL-terminated URL path.

Returns: Whether the rule matches path.

Return values

true	The rule matches.
false	The rule does not match.

Precondition: pat and path are non-NULL.; pat is readable for len bytes.

Postcondition: Inputs are unchanged.; No state is modified.

Note: Thread-safe: reads only arguments.

Since: 0.1.0

Definition at line 548 of file mdl_robots.c.

References internal_glob_prefix(), and RA8_INTERNAL.

Referenced by internal_best_matching_rule().

◆ internal_scan_spec()

bool internal_scan_spec	(	const char *	text,
		size_t	len,
		const char *	ua_token,
		int *	best,
		bool *	wild )

static

Pass 1: find the best specific match length and wildcard presence.

Scans every User-agent field without retaining rules. The longest case-insensitive prefix match becomes best; a * group is reported independently through wild for use only when no specific group wins.

Parameters

[in]	text	Robots document bytes.
[in]	len	Number of readable bytes at `text`.
[in]	ua_token	Product token to match against user-agent fields.
[out]	best	Longest specific match, or the no-match sentinel.
[out]	wild	Whether a wildcard user-agent group was present.

Returns: Whether every input line fit the bounded parser buffer.

Return values

true	The complete document was scanned without truncation.
false	At least one line exceeded the parser buffer.

Precondition: text and ua_token are non-NULL; text is readable for len bytes.; best and wild point to writable result storage.

Postcondition: best and wild describe all complete and truncated line prefixes scanned.; The input document and user-agent token are unchanged.

Note: A false result makes the enclosing parse invalid rather than silently permissive.

Since: 0.1.0

Definition at line 249 of file mdl_robots.c.

References internal_agent_spec(), internal_ieq(), internal_next_line(), internal_split_field(), k_robots_line_max, and k_spec_none.

Referenced by mdl_robots_parse().

◆ internal_set_crawl()

void internal_set_crawl	(	mdl_robots_t *	out,
		const char *	value )

static

Record the strictest Crawl-delay seen (clamped to the ceiling).

Parses a finite non-negative decimal and retains the largest bounded delay.

Parameters

[in,out]	out	Parsed robots result.
[in]	value	NUL-terminated crawl-delay value.

Returns: Nothing.

Precondition: out and value are non-NULL.; out contains writable result storage.

Postcondition: Valid input can only maintain or increase crawl_delay_ms.; Invalid text leaves the delay unchanged.

Note: Not thread-safe: may mutate out.

Since: 0.1.0

Definition at line 345 of file mdl_robots.c.

References mdl_robots_t::crawl_delay_ms, mdl_robots_t::have_crawl_delay, k_crawl_cap_ms, k_ms_per_s, and RA8_INTERNAL.

Referenced by internal_apply_directive().

◆ internal_split_field()

bool internal_split_field	(	char *	line,
		char **	field,
		char **	value )

static

Split a trimmed line into field/value on the first :.

Replaces the delimiter with NUL and returns trimmed pointers into line.

Parameters

[in,out]	line	Writable directive line.
[out]	field	Receives the field pointer.
[out]	value	Receives the value pointer.

Returns: Whether a non-empty field and delimiter were found.

Return values

true	Both outputs identify substrings in `line`.
false	No delimiter exists or the field is empty.

Precondition: All arguments are non-NULL.; line is writable and NUL-terminated.

Postcondition: On true, line is split in place.; Without a delimiter, outputs are unchanged.

Note: Returned pointers borrow line storage.

Since: 0.1.0

Definition at line 188 of file mdl_robots.c.

References internal_trim_ws(), RA8_INTERNAL, and strchr().

Referenced by internal_harvest(), and internal_scan_spec().

◆ internal_trim_ws()

char * internal_trim_ws ( char * s )

static

Trim leading/trailing ASCII whitespace in place; return start.

Definition at line 108 of file mdl_robots.c.

References RA8_INTERNAL, and strlen().

Referenced by internal_split_field().

◆ mdl_robots_allows()

bool mdl_robots_allows	(	const mdl_robots_t *	robots,
		const char *	path )

Decide whether path is allowed by a parsed robots group.

Applies the longest-match rule; when an Allow and a Disallow match at the same pattern length, the Allow wins (RFC 9309). A group with no rules permits everything. Patterns honour * and a trailing $.

Parameters

[in]	robots	Parsed rules from mdl_robots_parse.
[in]	path	URL path to test (with leading /).

Returns: Whether a request for path is permitted.

Return values

true	No rule matches, or the winning rule is an Allow.
false	The winning (longest) matching rule is a Disallow.

Precondition: robots and path are non-NULL.; path begins with / (a URL path, not a full URL).

Postcondition: Neither argument is modified.; A tie at the longest match is resolved in favour of Allow.

Note: Thread-safe: depends only on its arguments.

Since: 0.1.0

Definition at line 578 of file mdl_robots.c.

References internal_best_matching_rule(), k_mdl_rule_allow, mdl_robots_rule_t::kind, and mdl_robots_t::valid.

◆ mdl_robots_cache_consult()

const mdl_robots_t * mdl_robots_cache_consult	(	mdl_robots_cache_t *	cache,
		const char *	scheme,
		const char *	host,
		const char *	ua_token,
		mdl_robots_fetch_fn	fetch,
		void *	ctx,
		char *	scratch,
		size_t	scratch_cap )

Return the cached robots rules for host, fetching on first contact.

On a cache miss, builds <scheme>://<host>/robots.txt, calls fetch, applies the RFC 9309 convention (absent/transport error = allow all; 5xx = disallow all), parses a retrieved body for ua_token, and stores the outcome. Subsequent calls for the same host return the stored entry without a network round-trip. When the cache is full, the fetched result is returned uncached rather than evicting a live entry.

Parameters

[in]	cache	Per-run cache (zero-initialised before first use).
[in]	scheme	URL scheme ("http" or "https").
[in]	host	Host to consult.
[in]	ua_token	Our user-agent product token.
[in]	fetch	Fetch callback (dependency injection seam).
[in]	ctx	Opaque context passed to fetch.
[out]	scratch	Working buffer the callback fills.
[in]	scratch_cap	Capacity of scratch.

Returns: Borrowed pointer to the host's parsed rules (owned by cache), or NULL when disallow_all applies (caller refuses every path).

Return values

non-NULL	Rules to test paths against.
NULL	The host said "disallow all" (5xx); refuse every path.

Precondition: Every pointer argument is non-NULL; scratch_cap > 0.; scheme is "http" or "https".

Postcondition: A cache entry for host exists unless the cache was full.; scratch may be overwritten.

Note: Not thread-safe: mutates cache.

Since: 0.1.0

Definition at line 620 of file mdl_robots.c.

References mdl_robots_cache_entry_t::disallow_all, mdl_robots_cache_entry_t::host, internal_cache_find(), internal_cache_slot(), k_mdl_robots_fetch_denied, k_mdl_robots_fetch_ok, k_mdl_robots_host_max, k_robots_url_max, mdl_robots_parse(), nullptr, mdl_robots_cache_entry_t::rules, mdl_robots_cache_entry_t::scheme, strcmp(), strnlen(), mdl_robots_cache_entry_t::used, and mdl_robots_t::valid.

Referenced by mdl_session_url_allowed().

◆ mdl_robots_disallow_reason()

const char * mdl_robots_disallow_reason	(	const mdl_robots_t *	robots,
		const char *	path )

Return the Disallow pattern that forbids path, or NULL if allowed.

Names the exact rule responsible for a refusal so the caller can log a clear, actionable message rather than a bare "disallowed". Uses the same longest-match, Allow-wins-ties resolution as mdl_robots_allows.

Parameters

[in]	robots	Parsed rules from mdl_robots_parse.
[in]	path	URL path to test (with leading /).

Returns: The winning Disallow pattern, or NULL when path is allowed.

Return values

non-NULL	Borrowed pointer to the blocking rule's pattern.
NULL	No rule blocks path (the winning rule allows, or none).

Precondition: robots and path are non-NULL.; path begins with /.

Postcondition: Neither argument is modified; the result is owned by robots.

Note: Thread-safe: depends only on its arguments.

Since: 0.1.0

Definition at line 587 of file mdl_robots.c.

References internal_best_matching_rule(), k_mdl_rule_disallow, mdl_robots_rule_t::kind, and mdl_robots_rule_t::path.

Referenced by mdl_session_url_allowed().

◆ mdl_robots_parse()

void mdl_robots_parse	(	const char *	text,
		size_t	len,
		const char *	ua_token,
		mdl_robots_t *	out )

Parse robots.txt text for the group matching ua_token.

Selects the most specific matching User-agent group (a case-insensitive User-agent value that is a prefix of ua_token; * is the fallback), merging groups that match at the same specificity, and collects that group's Allow/Disallow rules and the strictest Crawl-delay. Empty Disallow values (which impose no restriction) are skipped. Malformed lines are ignored. An input with no matching group leaves out empty (allow all).

Parameters

[in]	text	robots.txt bytes (need not be NUL-terminated).
[in]	len	Length of text in bytes.
[in]	ua_token	Our user-agent product token (e.g. "mdl").
[out]	out	Result; fully overwritten (zeroed first).

Returns: Nothing.

Precondition: ua_token and out are non-NULL; text is non-NULL when len > 0.; out points to writable storage of sizeof(mdl_robots_t).

Postcondition: out->count <= k_mdl_robots_max_rules.; out is fully initialised even when no group matches.

Note: Thread-safe: writes only out.

Since: 0.1.0

Definition at line 450 of file mdl_robots.c.

References internal_harvest(), internal_scan_spec(), k_spec_none, and mdl_robots_t::valid.

Referenced by mdl_robots_cache_consult().

Enumerations

Functions

Detailed Description

Enumeration Type Documentation

◆ mdl_robots_local_size_t

◆ mdl_robots_ms_t

◆ mdl_robots_spec_t

Function Documentation

◆ internal_add_rule()

◆ internal_agent_spec()

◆ internal_apply_directive()

◆ internal_best_matching_rule()

◆ internal_cache_find()

◆ internal_cache_slot()

◆ internal_ci_prefix()

◆ internal_glob_prefix()

◆ internal_group_selected()

◆ internal_harvest()

◆ internal_ieq()

◆ internal_lower_ascii()

◆ internal_next_line()

◆ internal_robots_match()

◆ internal_scan_spec()

◆ internal_set_crawl()

◆ internal_split_field()

◆ internal_trim_ws()

◆ mdl_robots_allows()

◆ mdl_robots_cache_consult()

◆ mdl_robots_disallow_reason()

◆ mdl_robots_parse()