reflow_tokenize_lex.c

Go to the documentation of this file.

 
#include <stddef.h>
#include <stdint.h>
#include <string.h>
 
#include "ra8_attributes.h"
#include "ra8_err.h"
#include "reflow.h"
#include "reflow_tokenize_internal.h"
 
/* ===========================================================================
 * Public (test-exposed) helpers -- documented in reflow_tokenize_internal.h
 * ===========================================================================
 */
 
bool priv_reflow_tok_is_xml_whitespace(char c)
{
  return (c == ' ') || (c == '\t') || (c == '\n') || (c == '\r') || (c == '\f') || (c == '\v');
}
 
size_t priv_reflow_tok_utf8_encode(uint32_t cp, uint8_t* dst)
{
  uint32_t value = cp;
  if (value > (uint32_t)k_priv_uc_max) {
    value = (uint32_t)k_priv_uc_max;
  }
  if (value < (uint32_t)k_priv_uc_2byte) {
    dst[0] = (uint8_t)value;
    return 1U;
  }
  if (value < (uint32_t)k_priv_uc_3byte) {
    dst[0] = (uint8_t)((uint32_t)k_priv_utf8_lead2 | (value >> (uint32_t)k_priv_utf8_sh6));
    dst[1] = (uint8_t)((uint32_t)k_priv_utf8_cont | (value & (uint32_t)k_priv_utf8_mask));
    return 2U;
  }
  if (value < (uint32_t)k_priv_uc_4byte) {
    dst[0] = (uint8_t)((uint32_t)k_priv_utf8_lead3 | (value >> (uint32_t)k_priv_utf8_sh12));
    dst[1] = (uint8_t)((uint32_t)k_priv_utf8_cont |
                       ((value >> (uint32_t)k_priv_utf8_sh6) & (uint32_t)k_priv_utf8_mask));
    dst[2] = (uint8_t)((uint32_t)k_priv_utf8_cont | (value & (uint32_t)k_priv_utf8_mask));
    return 3U;
  }
  dst[0] = (uint8_t)((uint32_t)k_priv_utf8_lead4 | (value >> (uint32_t)k_priv_utf8_sh18));
  dst[1] = (uint8_t)((uint32_t)k_priv_utf8_cont |
                     ((value >> (uint32_t)k_priv_utf8_sh12) & (uint32_t)k_priv_utf8_mask));
  dst[2] = (uint8_t)((uint32_t)k_priv_utf8_cont |
                     ((value >> (uint32_t)k_priv_utf8_sh6) & (uint32_t)k_priv_utf8_mask));
  dst[3] = (uint8_t)((uint32_t)k_priv_utf8_cont | (value & (uint32_t)k_priv_utf8_mask));
  return 4U;
}

RA8_INTERNAL
static size_t internal_local_lower(const char* name, size_t len, char* dst)
{
  size_t start = 0U;
  for (size_t k = 0U; k < len; ++k) {
    if (name[k] == ':') {
      start = k + 1U;
    }
  }
  size_t n = 0U;
  for (size_t k = start; (k < len) && ((n + 1U) < (size_t)k_priv_tag_name_cap); ++k) {
    char c = name[k];
    if ((c >= 'A') && (c <= 'Z')) {
      c = (char)(c + ('a' - 'A'));
    }
    dst[n] = c;
    ++n;
  }
  dst[n] = '\0';
  return n;
}
 
reflow_html_tag_t priv_reflow_tok_classify(const char* name, size_t len)
{
  if (name == nullptr) {
    return k_reflow_tag_unknown;
  }
  char lower[k_priv_tag_name_cap];
  (void)internal_local_lower(name, len, lower);
 
  static const struct {
    const char*       word; 
    reflow_html_tag_t tag;  
  } k_map[] = {
    {"p", k_reflow_tag_p},
    {"h1", k_reflow_tag_h1},
    {"h2", k_reflow_tag_h2},
    {"h3", k_reflow_tag_h3},
    {"h4", k_reflow_tag_h4},
    {"h5", k_reflow_tag_h5},
    {"h6", k_reflow_tag_h6},
    {"em", k_reflow_tag_em},
    {"strong", k_reflow_tag_strong},
    {"b", k_reflow_tag_b},
    {"i", k_reflow_tag_i},
    {"br", k_reflow_tag_br},
    {"hr", k_reflow_tag_hr},
    {"ul", k_reflow_tag_ul},
    {"ol", k_reflow_tag_ol},
    {"li", k_reflow_tag_li},
    {"blockquote", k_reflow_tag_blockquote},
    {"a", k_reflow_tag_a},
    {"img", k_reflow_tag_img},
    {"link", k_reflow_tag_link},
    {"table", k_reflow_tag_table},
    {"tr", k_reflow_tag_tr},
    {"td", k_reflow_tag_td},
    {"th", k_reflow_tag_th},
  };
  for (size_t k = 0U; k < (sizeof(k_map) / sizeof(k_map[0])); ++k) {
    if (strcmp(lower, k_map[k].word) == 0) {
      return k_map[k].tag;
    }
  }
  return k_reflow_tag_unknown;
}

RA8_INTERNAL
static bool
internal_decode_numeric(const char* src, size_t avail, uint32_t* out_cp, size_t* out_used)
{
  size_t   i    = 2U; /* past "&#" */
  uint32_t base = (uint32_t)k_priv_base_dec;
  /* mcdc-deactivated: the sole caller priv_reflow_tok_decode_entity guarantees window >= k_priv_entity_min (4) before delegating, so i == 2 < avail always holds; the (i < avail) bound cannot be flipped false on any public path. */
  if ((i < avail) && ((src[i] == 'x') || (src[i] == 'X'))) {
    base = (uint32_t)k_priv_base_hex;
    ++i;
  }
  uint32_t cp     = 0U;
  size_t   digits = 0U;
  while ((i < avail) && (src[i] != ';')) {
    char     c = src[i];
    uint32_t d; /* assigned on every non-returning path below */
    if ((c >= '0') && (c <= '9')) {
      d = (uint32_t)(uint8_t)c - (uint32_t)(uint8_t)'0';
    } else if ((base == (uint32_t)k_priv_base_hex) && (c >= 'a') && (c <= 'f')) {
      d = ((uint32_t)(uint8_t)c - (uint32_t)(uint8_t)'a') + (uint32_t)k_priv_hex_offset;
    } else if ((base == (uint32_t)k_priv_base_hex) && (c >= 'A') && (c <= 'F')) {
      d = ((uint32_t)(uint8_t)c - (uint32_t)(uint8_t)'A') + (uint32_t)k_priv_hex_offset;
    } else {
      return false;
    }
    cp = (cp * base) + d;
    ++digits;
    ++i;
  }
  /* mcdc-deactivated: the scan loop above exits with i < avail only when src[i] == ';', so (src[i] != ';') is co-determined by (i >= avail) and can never independently flip; its independence pair is structurally unreachable. */
  if ((digits == 0U) || (i >= avail) || (src[i] != ';')) {
    return false;
  }
  *out_cp   = cp;
  *out_used = i + 1U;
  return true;
}
 
bool priv_reflow_tok_decode_entity(const char* src,
                                   size_t      avail,
                                   uint32_t*   out_cp,
                                   size_t*     out_used)
{
  const size_t window =
    (avail < (size_t)k_priv_entity_window) ? avail : (size_t)k_priv_entity_window;
  if ((window < (size_t)k_priv_entity_min) || (src[1] == '\0')) {
    return false;
  }
  if (src[1] == '#') {
    return internal_decode_numeric(src, window, out_cp, out_used);
  }
  static const struct {
    const char* word; 
    uint32_t    cp;   
  } k_named[] = {
    {"amp", (uint32_t)'&'},
    {"lt", (uint32_t)'<'},
    {"gt", (uint32_t)'>'},
    {"quot", (uint32_t)'"'},
    {"apos", (uint32_t)'\''},
  };
  for (size_t e = 0U; e < (sizeof(k_named) / sizeof(k_named[0])); ++e) {
    const size_t wlen = strlen(k_named[e].word);
    if (((wlen + 2U) <= window) && (src[1U + wlen] == ';') &&
        (strncmp(&src[1], k_named[e].word, wlen) == 0)) {
      *out_cp   = k_named[e].cp;
      *out_used = wlen + 2U;
      return true;
    }
  }
  return false;
}
 
/* ===========================================================================
 * Block / inline-style classification
 * ===========================================================================
 */

bool priv_reflow_tok_is_block(reflow_html_tag_t tag)
{
  switch (tag) {
    case k_reflow_tag_p:
    case k_reflow_tag_h1:
    case k_reflow_tag_h2:
    case k_reflow_tag_h3:
    case k_reflow_tag_h4:
    case k_reflow_tag_h5:
    case k_reflow_tag_h6:
    case k_reflow_tag_ul:
    case k_reflow_tag_ol:
    case k_reflow_tag_li:
    case k_reflow_tag_blockquote:
    case k_reflow_tag_table:
    case k_reflow_tag_tr:
    case k_reflow_tag_td:
    case k_reflow_tag_th:
      return true;
    default:
      return false;
  }
}

uint8_t priv_reflow_tok_style_for(reflow_html_tag_t tag)
{
  switch (tag) {
    case k_reflow_tag_b:
    case k_reflow_tag_strong:
      return (uint8_t)k_reflow_style_bold;
    case k_reflow_tag_i:
    case k_reflow_tag_em:
      return (uint8_t)k_reflow_style_italic;
    case k_reflow_tag_a:
      return (uint8_t)k_reflow_style_underline;
    default:
      return 0U;
  }
}
 
/* ===========================================================================
 * Token-pool emit + text-pool feed / stash + literal scanners
 * ===========================================================================
 */

bool priv_reflow_tok_emit(reflow_t*           engine,
                          reflow_token_kind_t kind,
                          reflow_html_tag_t   tag,
                          uint8_t             style,
                          uint32_t            off,
                          uint32_t            len)
{
  if (engine->token_count >= (uint32_t)k_reflow_max_tokens) {
    return false;
  }
  reflow_token_t* tok = &engine->tokens[engine->token_count];
  tok->kind           = (uint8_t)kind;
  tok->tag            = (uint8_t)tag;
  tok->style          = style;
  tok->reserved       = 0U;
  tok->text_off       = off;
  tok->text_len       = len;
  tok->color          = (uint32_t)k_reflow_color_inherit;
  tok->css_font_px    = 0U;
  tok->reserved16     = 0U;
  engine->token_count++;
  return true;
}

bool priv_reflow_tok_feed(reflow_t* engine, char ch, bool* last_ws)
{
  char value = ch;
  if (priv_reflow_tok_is_xml_whitespace(value)) {
    if (*last_ws) {
      return true;
    }
    value    = ' ';
    *last_ws = true;
  } else {
    *last_ws = false;
  }
  if (engine->text_pool_used >= (uint32_t)k_reflow_text_pool_bytes) {
    return false;
  }
  engine->text_pool[engine->text_pool_used] = (uint8_t)value;
  engine->text_pool_used++;
  return true;
}

RA8_INTERNAL
static bool internal_feed_utf8(reflow_t* engine, uint32_t cp, bool* last_ws)
{
  uint8_t      enc[4];
  const size_t n = priv_reflow_tok_utf8_encode(cp, enc);
  for (size_t b = 0U; b < n; ++b) {
    if (!priv_reflow_tok_feed(engine, (char)enc[b], last_ws)) {
      return false;
    }
  }
  return true;
}

RA8_INTERNAL
static bool internal_stash_one(reflow_t*      engine,
                               const uint8_t* buf,
                               size_t         i,
                               size_t         end,
                               bool*          last_ws,
                               size_t*        consumed)
{
  if (buf[i] == (uint8_t)'&') {
    uint32_t cp   = 0U;
    size_t   used = 0U;
    if (priv_reflow_tok_decode_entity((const char*)&buf[i], end - i, &cp, &used)) {
      *consumed = used;
      return internal_feed_utf8(engine, cp, last_ws);
    }
  }
  *consumed = 1U;
  return priv_reflow_tok_feed(engine, (char)buf[i], last_ws);
}

bool priv_reflow_tok_stash_run(reflow_t*      engine,
                               const uint8_t* buf,
                               size_t         start,
                               size_t         end,
                               uint32_t*      out_off,
                               uint32_t*      out_len)
{
  *out_off       = engine->text_pool_used;
  bool   last_ws = true; /* drop leading whitespace */
  size_t i       = start;
  while (i < end) {
    size_t consumed = 0U;
    if (!internal_stash_one(engine, buf, i, end, &last_ws, &consumed)) {
      return false;
    }
    i += consumed;
  }
  *out_len = engine->text_pool_used - *out_off;
  return true;
}

RA8_INTERNAL
static bool internal_bytes_equal(const uint8_t* buf, const char* lit, size_t len)
{
  bool equal = true;
  for (size_t k = 0U; (k < len) && equal; ++k) {
    equal = buf[k] == (uint8_t)lit[k];
  }
  return equal;
}
 
bool priv_reflow_tok_starts_with(const uint8_t* buf, size_t i, size_t len, const char* lit)
{
  const size_t n = strlen(lit);
  if ((i + n) > len) {
    return false;
  }
  return internal_bytes_equal(&buf[i], lit, n);
}

size_t priv_reflow_tok_skip_past(const uint8_t* buf, size_t i, size_t len, const char* lit)
{
  const size_t n   = strlen(lit);
  size_t       pos = i;
  while ((pos + n) <= len) {
    if (internal_bytes_equal(&buf[pos], lit, n)) {
      return pos + n;
    }
    ++pos;
  }
  return len;
}

size_t priv_reflow_tok_find_lit(const uint8_t* buf, size_t i, size_t len, const char* lit)
{
  const size_t n   = strlen(lit);
  size_t       pos = i;
  while ((pos + n) <= len) {
    if (internal_bytes_equal(&buf[pos], lit, n)) {
      return pos;
    }
    ++pos;
  }
  return len;
}