mdl_robots.c

Go to the documentation of this file.

#include "mdl_robots.h"
 
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
 
#include "ra8_attributes.h"

typedef enum : uint16_t {
  k_robots_line_max = 512, 
  k_robots_url_max  = 256, 
} mdl_robots_local_size_t;

typedef enum : uint32_t {
  k_ms_per_s     = 1000U,  
  k_crawl_cap_ms = 60000U, 
} mdl_robots_ms_t;

typedef enum : int8_t {
  k_spec_none = -1, 
} mdl_robots_spec_t;

RA8_INTERNAL static char internal_lower_ascii(char c)
{
  return (char)(((c >= 'A') && (c <= 'Z')) ? (c + ('a' - 'A')) : c);
}

RA8_INTERNAL static bool internal_ieq(const char* a, const char* b)
{
  size_t i = 0U;
  while ((a[i] != '\0') && (internal_lower_ascii(a[i]) == internal_lower_ascii(b[i]))) {
    ++i;
  }
  return internal_lower_ascii(a[i]) == internal_lower_ascii(b[i]);
}

RA8_INTERNAL static bool internal_ci_prefix(const char* prefix, const char* s)
{
  size_t i = 0U;
  while (prefix[i] != '\0') {
    if (internal_lower_ascii(s[i]) != internal_lower_ascii(prefix[i])) {
      return false;
    }
    ++i;
  }
  return true;
}

RA8_INTERNAL static char* internal_trim_ws(char* s)
{
  while ((*s == ' ') || (*s == '\t') || (*s == '\r')) {
    ++s;
  }
  size_t n = strlen(s);
  while ((n > 0U) && ((s[n - 1U] == ' ') || (s[n - 1U] == '\t') || (s[n - 1U] == '\r'))) {
    s[n - 1U] = '\0';
    --n;
  }
  return s;
}

RA8_INTERNAL static bool
internal_next_line(const char** pp, const char* end, char* buf, size_t cap, bool* truncated)
{
  const char* p = *pp;
  if (p >= end) {
    return false;
  }
  size_t n = 0U;
  while ((p < end) && (*p != '\n')) {
    if ((n + 1U) < cap) {
      buf[n] = *p;
      ++n;
    } else {
      *truncated = true;
    }
    ++p;
  }
  if (p < end) {
    ++p; /* consume the '\n' */
  }
  *pp        = p;
  buf[n]     = '\0';
  char* hash = strchr(buf, '#');
  if (hash != nullptr) {
    *hash = '\0';
  }
  return true;
}

RA8_INTERNAL static bool internal_split_field(char* line, char** field, char** value)
{
  char* colon = strchr(line, ':');
  if (colon == nullptr) {
    return false;
  }
  *colon = '\0';
  *field = internal_trim_ws(line);
  *value = internal_trim_ws(colon + 1);
  return (*field)[0] != '\0';
}

RA8_INTERNAL static int internal_agent_spec(const char* agent, const char* ua_token)
{
  if (strcmp(agent, "*") == 0) {
    return 0;
  }
  if ((agent[0] != '\0') && internal_ci_prefix(agent, ua_token)) {
    return (int)strlen(agent);
  }
  return (int)k_spec_none;
}

RA8_INTERNAL static bool
internal_scan_spec(const char* text, size_t len, const char* ua_token, int* best, bool* wild)
{
  *best           = (int)k_spec_none;
  *wild           = false;
  const char* p   = text;
  const char* end = text + len;
  char        line[k_robots_line_max];
  bool        truncated = false;
  while (internal_next_line(&p, end, line, sizeof(line), &truncated)) {
    char* field = nullptr;
    char* value = nullptr;
    if (!internal_split_field(line, &field, &value)) {
      continue;
    }
    if (internal_ieq(field, "user-agent")) {
      const int spec = internal_agent_spec(value, ua_token);
      if (spec == 0) {
        *wild = true;
      } else if (spec > *best) {
        *best = spec;
      }
    }
  }
  return !truncated;
}

RA8_INTERNAL static bool
internal_group_selected(int target, bool wildcard_target, int grp_spec, bool grp_wild)
{
  return wildcard_target ? grp_wild : ((target >= 0) && (grp_spec == target));
}

RA8_INTERNAL static void
internal_add_rule(mdl_robots_t* out, mdl_robots_rule_kind_t kind, const char* value)
{
  if ((value[0] == '\0') || (out->count >= (size_t)k_mdl_robots_max_rules)) {
    if (value[0] != '\0') {
      out->valid = false;
    }
    return; /* empty pattern imposes no restriction; full table stops growing */
  }
  if (strnlen(value, k_mdl_robots_path_max) >= k_mdl_robots_path_max) {
    out->valid = false;
    return;
  }
  mdl_robots_rule_t* r = &out->rules[out->count];
  r->kind              = kind;
  (void)snprintf(r->path, sizeof(r->path), "%s", value);
  r->len = (uint16_t)strlen(r->path);
  out->count++;
}

RA8_INTERNAL static void internal_set_crawl(mdl_robots_t* out, const char* value)
{
  char*        end  = nullptr;
  const double secs = strtod(value, &end);
  if ((end == value) || (*end != '\0') || !isfinite(secs) || (secs <= 0.0)) {
    return;
  }
  uint32_t ms = (secs >= ((double)k_crawl_cap_ms / (double)k_ms_per_s))
                  ? (uint32_t)k_crawl_cap_ms
                  : (uint32_t)(secs * (double)k_ms_per_s);
  if (!out->have_crawl_delay || (ms > out->crawl_delay_ms)) {
    out->crawl_delay_ms = ms;
  }
  out->have_crawl_delay = true;
}

RA8_INTERNAL static void
internal_apply_directive(mdl_robots_t* out, const char* field, const char* value)
{
  if (internal_ieq(field, "disallow")) {
    internal_add_rule(out, k_mdl_rule_disallow, value);
  } else if (internal_ieq(field, "allow")) {
    internal_add_rule(out, k_mdl_rule_allow, value);
  } else if (internal_ieq(field, "crawl-delay")) {
    internal_set_crawl(out, value);
  }
}

RA8_INTERNAL static void internal_harvest(const char*   text,
                                          size_t        len,
                                          const char*   ua_token,
                                          int           target,
                                          bool          wildcard_target,
                                          mdl_robots_t* out)
{
  const char* p          = text;
  const char* end        = text + len;
  bool        prev_agent = false;
  bool        selected   = false;
  int         grp_spec   = (int)k_spec_none;
  bool        grp_wild   = false;
  char        line[k_robots_line_max];
  bool        truncated = false;
  while (internal_next_line(&p, end, line, sizeof(line), &truncated)) {
    char* field = nullptr;
    char* value = nullptr;
    if (!internal_split_field(line, &field, &value)) {
      continue;
    }
    if (internal_ieq(field, "user-agent")) {
      if (!prev_agent) {
        grp_spec = (int)k_spec_none;
        grp_wild = false;
      }
      const int spec = internal_agent_spec(value, ua_token);
      if (spec == 0) {
        grp_wild = true;
      } else if (spec > grp_spec) {
        grp_spec = spec;
      }
      selected   = internal_group_selected(target, wildcard_target, grp_spec, grp_wild);
      prev_agent = true;
    } else {
      prev_agent = false;
      if (selected) {
        internal_apply_directive(out, field, value);
      }
    }
  }
  if (truncated) {
    out->valid = false;
  }
}
 
void mdl_robots_parse(const char* text, size_t len, const char* ua_token, mdl_robots_t* out)
{
  if (out == nullptr) {
    return;
  }
  *out       = (mdl_robots_t){};
  out->valid = true;
  if ((text == nullptr) && (len != 0U)) {
    out->valid = false;
    return;
  }
  if (ua_token == nullptr) {
    out->valid = false;
    return;
  }
  if (len == 0U) {
    return;
  }
  int  best = (int)k_spec_none;
  bool wild = false;
  if (!internal_scan_spec(text, len, ua_token, &best, &wild)) {
    out->valid = false;
    return;
  }
  if ((best < 0) && !wild) {
    return; /* no group matches us -> no restrictions */
  }
  const bool wildcard_target = (best < 0) && wild;
  internal_harvest(text, len, ua_token, best, wildcard_target, out);
}

RA8_INTERNAL static bool
internal_glob_prefix(const char* pat, size_t patlen, const char* path, bool anchored)
{
  size_t      pi   = 0U;
  const char* s    = path;
  size_t      star = patlen + 1U; /* > patlen means "no star seen yet" */
  const char* ss   = nullptr;
  while (*s != '\0') {
    if ((pi < patlen) && (pat[pi] == '*')) {
      star = pi;
      ++pi;
      ss = s;
    } else if ((pi < patlen) && (pat[pi] == *s)) {
      ++pi;
      ++s;
    } else if ((star <= patlen) && (ss != nullptr)) {
      /* star <= patlen holds only after `ss = s` ran, so ss is non-NULL here;
       * the explicit check makes that invariant visible to the analyser. */
      pi = star + 1U;
      ++ss;
      s = ss;
    } else {
      return false;
    }
    if ((pi == patlen) && !anchored) {
      return true; /* whole pattern consumed -> prefix match */
    }
  }
  while ((pi < patlen) && (pat[pi] == '*')) {
    ++pi;
  }
  return pi == patlen;
}

RA8_INTERNAL static bool internal_robots_match(const char* pat, size_t len, const char* path)
{
  bool anchored = false;
  if ((len > 0U) && (pat[len - 1U] == '$')) {
    anchored = true;
    --len;
  }
  return internal_glob_prefix(pat, len, path, anchored);
}

RA8_INTERNAL static const mdl_robots_rule_t* internal_best_matching_rule(const mdl_robots_t* robots,
                                                                         const char*         path)
{
  const mdl_robots_rule_t* best     = nullptr;
  int                      best_len = (int)k_spec_none;
  for (size_t i = 0U; i < robots->count; ++i) {
    const mdl_robots_rule_t* r = &robots->rules[i];
    if (!internal_robots_match(r->path, r->len, path)) {
      continue;
    }
    const bool is_allow = (r->kind == k_mdl_rule_allow);
    if (((int)r->len > best_len) || (((int)r->len == best_len) && is_allow)) {
      best_len = (int)r->len;
      best     = r;
    }
  }
  return best;
}
 
bool mdl_robots_allows(const mdl_robots_t* robots, const char* path)
{
  if ((robots == nullptr) || (path == nullptr) || !robots->valid) {
    return false;
  }
  const mdl_robots_rule_t* best = internal_best_matching_rule(robots, path);
  return (best == nullptr) || (best->kind == k_mdl_rule_allow);
}
 
const char* mdl_robots_disallow_reason(const mdl_robots_t* robots, const char* path)
{
  if ((robots == nullptr) || (path == nullptr)) {
    return nullptr;
  }
  const mdl_robots_rule_t* best = internal_best_matching_rule(robots, path);
  return ((best != nullptr) && (best->kind == k_mdl_rule_disallow)) ? best->path : nullptr;
}

RA8_INTERNAL static mdl_robots_cache_entry_t*
internal_cache_find(mdl_robots_cache_t* cache, const char* scheme, const char* host)
{
  for (size_t i = 0U; i < (size_t)k_mdl_robots_max_hosts; ++i) {
    if (cache->hosts[i].used && (strcmp(cache->hosts[i].scheme, scheme) == 0) &&
        (strcmp(cache->hosts[i].host, host) == 0)) {
      return &cache->hosts[i];
    }
  }
  return nullptr;
}

RA8_INTERNAL static mdl_robots_cache_entry_t* internal_cache_slot(mdl_robots_cache_t* cache)
{
  for (size_t i = 0U; i < (size_t)k_mdl_robots_max_hosts; ++i) {
    if (!cache->hosts[i].used) {
      return &cache->hosts[i];
    }
  }
  return &cache->overflow;
}
 
const mdl_robots_t* mdl_robots_cache_consult(mdl_robots_cache_t* cache,
                                             const char*         scheme,
                                             const char*         host,
                                             const char*         ua_token,
                                             mdl_robots_fetch_fn fetch,
                                             void*               ctx,
                                             char*               scratch,
                                             size_t              scratch_cap)
{
  if ((cache == nullptr) || (scheme == nullptr) || (host == nullptr) || (ua_token == nullptr) ||
      (fetch == nullptr) || (scratch == nullptr) || (scratch_cap == 0U) ||
      ((strcmp(scheme, "http") != 0) && (strcmp(scheme, "https") != 0)) ||
      (strnlen(host, k_mdl_robots_host_max) >= k_mdl_robots_host_max)) {
    return nullptr;
  }
  mdl_robots_cache_entry_t* e = internal_cache_find(cache, scheme, host);
  if (e == nullptr) {
    e              = internal_cache_slot(cache);
    *e             = (mdl_robots_cache_entry_t){};
    e->rules.valid = true;
    (void)snprintf(e->scheme, sizeof(e->scheme), "%s", scheme);
    (void)snprintf(e->host, sizeof(e->host), "%s", host);
    char      url[k_robots_url_max];
    const int url_len = snprintf(url, sizeof(url), "%s://%s/robots.txt", scheme, host);
    if ((url_len < 0) || ((size_t)url_len >= sizeof(url))) {
      e->disallow_all = true;
      e->used         = true;
      return nullptr;
    }
    size_t                          got = 0U;
    const mdl_robots_fetch_result_t rc  = fetch(ctx, url, scratch, scratch_cap, &got);
    if (rc == k_mdl_robots_fetch_denied) {
      e->disallow_all = true;
    } else if (rc == k_mdl_robots_fetch_ok) {
      mdl_robots_parse(scratch, got, ua_token, &e->rules);
      if (!e->rules.valid) {
        e->disallow_all = true;
      }
    }
    e->used = true;
  }
  return e->disallow_all ? nullptr : &e->rules;
}