mdl_extract_hits.c

Go to the documentation of this file.

#include <string.h>
 
#include "mdl_extract.h"
#include "mdl_extract_internal.h"

typedef struct {
  const char* name; 
  char        ch;   
} mdl_entity_t;

RA8_INTERNAL static bool internal_is_ws(char c)
{
  return (c == ' ') || (c == '\t') || (c == '\n') || (c == '\r') || (c == '\f') || (c == '\v');
}

RA8_INTERNAL static bool
internal_decode_entity(const char* s, size_t len, size_t* i, char* out_ch, bool* space)
{
  static const mdl_entity_t table[] = {
    {"amp;", '&'},
    {"lt;", '<'},
    {"gt;", '>'},
    {"quot;", '"'},
    {"#39;", '\''},
    {"apos;", '\''},
    {"nbsp;", '\0'},
  };
  const size_t start = *i + 1U; /* byte after '&' */
  for (size_t e = 0U; e < (sizeof(table) / sizeof(table[0])); ++e) {
    const size_t nlen = strlen(table[e].name);
    if (((start + nlen) <= len) && (memcmp(s + start, table[e].name, nlen) == 0)) {
      *i      = start + nlen - 1U; /* leave *i on the ';'; the loop's ++ steps past */
      *space  = (table[e].ch == '\0');
      *out_ch = table[e].ch;
      return true;
    }
  }
  return false;
}

RA8_INTERNAL static void
internal_append_out_char(char* out, size_t* n, size_t cap, bool* fits, char ch)
{
  if ((*n + 1U) < cap) {
    out[*n] = ch;
    ++(*n);
  } else {
    *fits = false;
  }
}

RA8_INTERNAL static bool
internal_clean_inner_text(const char* text, size_t len, char* out, size_t cap)
{
  size_t n       = 0U;
  bool   pending = false; /* a space is owed before the next visible char */
  bool   fits    = true;
  for (size_t i = 0U; i < len; ++i) {
    const char c = text[i];
    if (c == '<') {
      while ((i < len) && (text[i] != '>')) {
        ++i; /* skip a nested tag such as <span> */
      }
      continue;
    }
    char emit  = c;
    bool space = internal_is_ws(c);
    if (c == '&') {
      char decoded = '\0';
      if (internal_decode_entity(text, len, &i, &decoded, &space)) {
        emit = decoded;
      }
    }
    if (space) {
      pending = (n > 0U); /* collapse; never a leading space */
      continue;
    }
    if (pending) {
      internal_append_out_char(out, &n, cap, &fits, ' ');
    }
    pending = false;
    internal_append_out_char(out, &n, cap, &fits, emit);
  }
  out[n] = '\0';
  return fits;
}

RA8_INTERNAL static void internal_url_slug(const char* url, char* out, size_t cap)
{
  const char*  q   = strpbrk(url, "?#");
  const size_t end = (q == nullptr) ? strlen(url) : (size_t)(q - url);
  size_t       beg = 0U;
  size_t       fin = end;
  while ((fin > 0U) && (url[fin - 1U] == '/')) {
    --fin; /* drop trailing slashes */
  }
  beg = fin;
  while ((beg > 0U) && (url[beg - 1U] != '/')) {
    --beg; /* walk back to the segment start */
  }
  size_t n = 0U;
  for (size_t i = beg; (i < fin) && ((n + 1U) < cap); ++i) {
    out[n] = url[i];
    ++n;
  }
  out[n] = '\0';
}

RA8_INTERNAL static bool internal_anchor_title(const char* html,
                                               size_t      html_len,
                                               const char* tag,
                                               size_t      tag_len,
                                               size_t      gt_off,
                                               char*       out,
                                               size_t      cap)
{
  if (internal_find_attr_value(tag, tag_len, "title", out, cap) && (out[0] != '\0')) {
    return true;
  }
  if (gt_off >= html_len) {
    return false;
  }
  const size_t text_off = gt_off + 1U;
  const char*  close    = internal_find_ci(html + text_off, html_len - text_off, "</a");
  const size_t text_len =
    (close == nullptr) ? (html_len - text_off) : (size_t)(close - (html + text_off));
  return internal_clean_inner_text(html + text_off, text_len, out, cap) && (out[0] != '\0');
}

RA8_INTERNAL static size_t internal_hit_index_of(const mdl_hit_list_t* out, const char* url)
{
  for (size_t i = 0U; i < out->count; ++i) {
    if (strcmp(out->hits[i].url, url) == 0) {
      return i;
    }
  }
  return out->count;
}

RA8_INTERNAL static ra8_err_t internal_merge_hit(mdl_hit_list_t* out,
                                                 bool*           real,
                                                 const char*     url,
                                                 const char*     title,
                                                 bool            title_real)
{
  const size_t at = internal_hit_index_of(out, url);
  if (at < out->count) {
    if (title_real && !real[at]) {
      (void)internal_copy_fits(out->hits[at].title, k_mdl_hit_title_max, title);
      real[at] = true;
    }
    return k_ra8_ok;
  }
  if (out->count >= (size_t)k_mdl_max_hits) {
    return k_ra8_err_no_mem;
  }
  (void)internal_copy_fits(out->hits[out->count].url, k_mdl_url_max, url);
  (void)internal_copy_fits(out->hits[out->count].title, k_mdl_hit_title_max, title);
  real[out->count] = title_real;
  out->count++;
  return k_ra8_ok;
}

RA8_INTERNAL static ra8_err_t internal_emit_hit(const char*     html,
                                                size_t          html_len,
                                                const char*     base_url,
                                                const char*     keep,
                                                const char*     tag,
                                                size_t          tag_len,
                                                size_t          gt_off,
                                                mdl_hit_list_t* out,
                                                bool*           real)
{
  char raw[k_mdl_url_max];
  if (!internal_find_attr_value(tag, tag_len, "href", raw, sizeof(raw))) {
    return k_ra8_ok; /* an <a> with no href is not a link */
  }
  char abs[k_mdl_url_max];
  if (!mdl_extract_resolve_url(base_url, raw, abs, sizeof(abs))) {
    return k_ra8_ok; /* unresolvable (e.g. javascript:, fragment): not counted
                      */
  }
  out->anchors_seen++;
  if (!internal_contains_ok(abs, keep)) {
    return k_ra8_ok; /* a real link, but not a result: counted, not stored */
  }
  char       title[k_mdl_hit_title_max];
  const bool title_real =
    internal_anchor_title(html, html_len, tag, tag_len, gt_off, title, sizeof(title));
  if (!title_real) {
    internal_url_slug(abs, title, sizeof(title));
  }
  return internal_merge_hit(out, real, abs, title, title_real);
}
 
ra8_err_t mdl_extract_hits(const char*     html,
                           size_t          html_len,
                           const char*     base_url,
                           const char*     url_contains,
                           mdl_hit_list_t* out)
{
  if ((html == nullptr) || (base_url == nullptr) || (out == nullptr)) {
    return k_ra8_err_invalid_arg;
  }
  out->count        = 0U;
  out->anchors_seen = 0U;
  bool real[k_mdl_max_hits];
  memset(real, 0, sizeof(real));
 
  size_t pos = 0U;
  while (pos < html_len) {
    const char* tag = internal_find_ci(html + pos, html_len - pos, "<a");
    if (tag == nullptr) {
      break;
    }
    const size_t tag_off = (size_t)(tag - html);
    const size_t after   = tag_off + (sizeof("<a") - 1U);
    if ((after < html_len) && !internal_is_name_end(html[after])) {
      pos = tag_off + 1U; /* "<article" while seeking "<a" */
      continue;
    }
    const char*  gt      = memchr(tag, '>', html_len - tag_off);
    const size_t gt_off  = (gt == nullptr) ? html_len : (size_t)(gt - html);
    const size_t tag_len = (gt == nullptr) ? (html_len - tag_off) : (size_t)(gt - tag);
    if (internal_emit_hit(html,
                          html_len,
                          base_url,
                          url_contains,
                          tag,
                          tag_len,
                          gt_off,
                          out,
                          real) == k_ra8_err_no_mem) {
      return k_ra8_err_no_mem;
    }
    pos = tag_off + ((gt == nullptr) ? tag_len : (tag_len + 1U));
  }
  return k_ra8_ok;
}

RA8_INTERNAL static bool internal_class_has_token(const char* classes, const char* value)
{
  const size_t value_len = strlen(value);
  const char*  at        = classes;
  while ((at = strstr(at, value)) != nullptr) {
    const bool left_ok  = (at == classes) || internal_is_ws(at[-1]);
    const char after    = at[value_len];
    const bool right_ok = (after == '\0') || internal_is_ws(after);
    if (left_ok && right_ok) {
      return true;
    }
    at += value_len;
  }
  return false;
}

RA8_INTERNAL static ra8_err_t internal_selector_copy(const char* value, char* out, size_t cap)
{
  if (value[0] == '\0') {
    return k_ra8_err_not_found;
  }
  if (!internal_copy_fits(out, cap, value)) {
    return k_ra8_err_invalid_size;
  }
  return k_ra8_ok;
}

RA8_INTERNAL static ra8_err_t
internal_extract_meta(const char* html, size_t len, const char* key, char* out, size_t cap)
{
  size_t pos = 0U;
  while (pos < len) {
    const char* tag = internal_find_ci(html + pos, len - pos, "<meta");
    if (tag == nullptr) {
      return k_ra8_err_not_found;
    }
    const size_t off = (size_t)(tag - html);
    const char*  gt  = memchr(tag, '>', len - off);
    if (gt == nullptr) {
      return k_ra8_err_not_found;
    }
    const size_t tag_len = (size_t)(gt - tag);
    char         name[k_mdl_hit_title_max];
    char         content[k_mdl_url_max];
    const bool   named = internal_find_attr_value(tag, tag_len, "property", name, sizeof(name)) ||
                         internal_find_attr_value(tag, tag_len, "name", name, sizeof(name));
    if (named && (strcmp(name, key) == 0) &&
        internal_find_attr_value(tag, tag_len, "content", content, sizeof(content))) {
      if (!internal_clean_inner_text(content, strlen(content), out, cap)) {
        return k_ra8_err_invalid_size;
      }
      return (out[0] == '\0') ? k_ra8_err_not_found : k_ra8_ok;
    }
    pos = off + tag_len + 1U;
  }
  return k_ra8_err_not_found;
}

RA8_INTERNAL static ra8_err_t
internal_extract_class(const char* html, size_t len, const char* token, char* out, size_t cap)
{
  size_t pos = 0U;
  while (pos < len) {
    const char* tag = memchr(html + pos, '<', len - pos);
    if (tag == nullptr) {
      return k_ra8_err_not_found;
    }
    const size_t off = (size_t)(tag - html);
    const char*  gt  = memchr(tag, '>', len - off);
    if (gt == nullptr) {
      return k_ra8_err_not_found;
    }
    const size_t tag_len = (size_t)(gt - tag);
    char         classes[k_mdl_hit_title_max];
    if (internal_find_attr_value(tag, tag_len, "class", classes, sizeof(classes)) &&
        internal_class_has_token(classes, token)) {
      const size_t inner_off = (size_t)(gt - html) + 1U;
      const char*  close     = internal_find_ci(html + inner_off, len - inner_off, "</");
      const size_t inner_len =
        (close == nullptr) ? (len - inner_off) : (size_t)(close - (html + inner_off));
      if (!internal_clean_inner_text(html + inner_off, inner_len, out, cap)) {
        return k_ra8_err_invalid_size;
      }
      return (out[0] == '\0') ? k_ra8_err_not_found : k_ra8_ok;
    }
    pos = off + tag_len + 1U;
  }
  return k_ra8_err_not_found;
}

RA8_INTERNAL static ra8_err_t
internal_extract_label(const char* html, size_t len, const char* label, char* out, size_t cap)
{
  const char* hit = internal_find_ci(html, len, label);
  if (hit == nullptr) {
    return k_ra8_err_not_found;
  }
  const size_t after_label = (size_t)(hit - html) + strlen(label);
  const char*  anchor      = internal_find_ci(html + after_label, len - after_label, "<a");
  if (anchor == nullptr) {
    return k_ra8_err_not_found;
  }
  const size_t anchor_off = (size_t)(anchor - html);
  const char*  gt         = memchr(anchor, '>', len - anchor_off);
  if (gt == nullptr) {
    return k_ra8_err_not_found;
  }
  const size_t inner_off = (size_t)(gt - html) + 1U;
  const char*  close     = internal_find_ci(html + inner_off, len - inner_off, "</a");
  const size_t inner_len =
    (close == nullptr) ? (len - inner_off) : (size_t)(close - (html + inner_off));
  if (!internal_clean_inner_text(html + inner_off, inner_len, out, cap)) {
    return k_ra8_err_invalid_size;
  }
  return (out[0] == '\0') ? k_ra8_err_not_found : k_ra8_ok;
}
 
ra8_err_t mdl_extract_selector(const char* html,
                               size_t      html_len,
                               const char* selector,
                               char*       out,
                               size_t      out_cap)
{
  if ((html == nullptr) || (selector == nullptr) || (out == nullptr) || (out_cap == 0U)) {
    return k_ra8_err_invalid_arg;
  }
  out[0]            = '\0';
  const char* value = strchr(selector, ':');
  if ((value == nullptr) || (value[1] == '\0')) {
    return k_ra8_err_invalid_arg;
  }
  ++value;
  if (strncmp(selector, "meta:", sizeof("meta:") - 1U) == 0) {
    return internal_extract_meta(html, html_len, value, out, out_cap);
  }
  if (strncmp(selector, "class:", sizeof("class:") - 1U) == 0) {
    return internal_extract_class(html, html_len, value, out, out_cap);
  }
  if (strncmp(selector, "label:", sizeof("label:") - 1U) == 0) {
    return internal_extract_label(html, html_len, value, out, out_cap);
  }
  if (strncmp(selector, "literal:", sizeof("literal:") - 1U) == 0) {
    return internal_selector_copy(value, out, out_cap);
  }
  return k_ra8_err_invalid_arg;
}