reflow_tokenize.c

Go to the documentation of this file.

 
#include <stddef.h>
#include <stdint.h>
#include <string.h>
 
#include "ra8_attributes.h"
#include "ra8_err.h"
#include "reflow.h"
#include "reflow_tokenize_internal.h"
 
/* ===========================================================================
 * Mutable tokenizer context (markup-engine private; lexical + attribute
 * primitives live in the companion TUs and never touch this struct).
 * ===========================================================================
 */

typedef struct {
  reflow_t* engine;      
  uint8_t   style;       
  uint8_t   active_link; 
  uint32_t  color;       
  uint16_t  css_font_px; 
  uint16_t  family_off;  
  uint16_t  family_len;  
  uint8_t   face_slot;   
  ra8_css_element_t stack_el[k_priv_max_depth];
  uint8_t           stack_style[k_priv_max_depth];   
  uint8_t           stack_link[k_priv_max_depth];    
  uint32_t          stack_color[k_priv_max_depth];   
  uint16_t          stack_font[k_priv_max_depth];    
  uint16_t          stack_fam_off[k_priv_max_depth]; 
  uint16_t          stack_fam_len[k_priv_max_depth]; 
  uint8_t           stack_face[k_priv_max_depth];    
  uint32_t          sp;                              
  uint32_t          suppress_sp;                     
  bool              saw_element;                     
} tok_ctx_t;
 
/* ===========================================================================
 * Internal flow helpers (context-aware)
 * ===========================================================================
 */

RA8_INTERNAL
static bool internal_suppressed(const tok_ctx_t* ctx)
{
  return ctx->suppress_sp != 0U;
}
 
/* ===========================================================================
 * Markup handlers (advance *pi)
 * ===========================================================================
 */

RA8_INTERNAL
static ra8_err_t internal_handle_cdata(tok_ctx_t* ctx, const uint8_t* buf, size_t* pi, size_t len)
{
  const size_t inner = *pi + strlen("<![CDATA[");
  size_t       close = inner;
  while (((close + 3U) <= len) &&
         !((buf[close] == (uint8_t)']') && (buf[close + 1U] == (uint8_t)']') &&
           (buf[close + 2U] == (uint8_t)'>'))) {
    ++close;
  }
  if ((ctx->sp > 0U) && !internal_suppressed(ctx)) {
    uint32_t off     = ctx->engine->text_pool_used;
    bool     last_ws = true;
    for (size_t k = inner; k < close; ++k) {
      if (!priv_reflow_tok_feed(ctx->engine, (char)buf[k], &last_ws)) {
        return k_ra8_err_no_mem;
      }
    }
    const uint32_t tlen = ctx->engine->text_pool_used - off;
    if ((tlen > 0U) && !priv_reflow_tok_emit(ctx->engine,
                                             k_reflow_tok_text,
                                             k_reflow_tag_unknown,
                                             ctx->style,
                                             off,
                                             tlen)) {
      return k_ra8_err_no_mem;
    }
    if (tlen > 0U) {
      ctx->engine->tokens[ctx->engine->token_count - 1U].reserved16 = (uint16_t)ctx->face_slot;
    }
  }
  *pi = ((close + 3U) <= len) ? (close + 3U) : len;
  return k_ra8_ok;
}

RA8_INTERNAL
static ra8_err_t internal_handle_end(tok_ctx_t* ctx, const uint8_t* buf, size_t* pi, size_t len)
{
  size_t i = *pi + 2U; /* past "</" */
  while ((i < len) && (buf[i] != (uint8_t)'>')) {
    ++i;
  }
  *pi = (i < len) ? (i + 1U) : len;
  if (ctx->sp == 0U) {
    return k_ra8_err_validation_failed;
  }
  ctx->sp--;
  const reflow_html_tag_t tag = (reflow_html_tag_t)ctx->stack_el[ctx->sp].tag;
  ctx->style                  = ctx->stack_style[ctx->sp];
  ctx->active_link            = ctx->stack_link[ctx->sp];
  ctx->color                  = ctx->stack_color[ctx->sp];
  ctx->css_font_px            = ctx->stack_font[ctx->sp];
  ctx->family_off             = ctx->stack_fam_off[ctx->sp];
  ctx->family_len             = ctx->stack_fam_len[ctx->sp];
  ctx->face_slot              = ctx->stack_face[ctx->sp];
  if (priv_reflow_tok_is_block(tag) && !internal_suppressed(ctx) &&
      !priv_reflow_tok_emit(ctx->engine, k_reflow_tok_block_end, tag, ctx->style, 0U, 0U)) {
    return k_ra8_err_no_mem;
  }
  /* Exited the display:none subtree once we pop back above its depth. */
  if ((ctx->suppress_sp != 0U) && (ctx->sp < ctx->suppress_sp)) {
    ctx->suppress_sp = 0U;
  }
  return k_ra8_ok;
}

RA8_INTERNAL
static reflow_html_tag_t
internal_parse_start(const uint8_t* buf, size_t* pi, size_t len, bool* selfclose)
{
  size_t       i      = *pi + 1U; /* past '<' */
  const size_t nstart = i;
  while ((i < len) && (buf[i] != (uint8_t)'>') && (buf[i] != (uint8_t)'/') &&
         !priv_reflow_tok_is_xml_whitespace((char)buf[i])) {
    ++i;
  }
  const reflow_html_tag_t tag = priv_reflow_tok_classify((const char*)&buf[nstart], i - nstart);
  *selfclose                  = false;
  while ((i < len) && (buf[i] != (uint8_t)'>')) {
    const char c = (char)buf[i];
    if ((c == '"') || (c == '\'')) {
      ++i;
      while ((i < len) && (buf[i] != (uint8_t)c)) {
        ++i;
      }
    } else if ((c == '/') && ((i + 1U) < len) && (buf[i + 1U] == (uint8_t)'>')) {
      *selfclose = true;
    } else {
      /* Continue scanning the current start tag. */
    }
    ++i;
  }
  *pi = (i < len) ? (i + 1U) : len;
  return tag;
}

RA8_INTERNAL
static ra8_err_t internal_open_attrs(tok_ctx_t*             ctx,
                                     const uint8_t*         tag,
                                     size_t                 span,
                                     reflow_html_tag_t      kind,
                                     bool                   block,
                                     const ra8_css_style_t* comp,
                                     uint16_t               css_font_px)
{
  if (internal_suppressed(ctx)) {
    return k_ra8_ok; /* inside a display:none subtree -> emit nothing */
  }
  if (block) {
    uint32_t id_off = 0U;
    uint32_t id_len = 0U;
    priv_reflow_tok_capture_attr(ctx->engine, tag, span, "id", sizeof("id") - 1U, &id_off, &id_len);
    if (!priv_reflow_tok_emit(ctx->engine,
                              k_reflow_tok_block_start,
                              kind,
                              ctx->style,
                              id_off,
                              id_len)) {
      return k_ra8_err_no_mem;
    }
    /* Stash the block's cascaded alignment + CSS font px on the block-start token
     * (left + 0 = unstyled defaults, so unstyled blocks lay out byte-identically). */
    const uint8_t   align = ((comp->set & (uint8_t)k_ra8_css_set_align) != 0U)
                              ? comp->align
                              : (uint8_t)k_reflow_align_left;
    reflow_token_t* bst   = &ctx->engine->tokens[ctx->engine->token_count - 1U];
    bst->reserved         = align;
    bst->css_font_px      = css_font_px;
    return k_ra8_ok;
  }
  if (kind == k_reflow_tag_a) {
    uint32_t href_off = 0U;
    uint32_t href_len = 0U;
    priv_reflow_tok_capture_attr(ctx->engine,
                                 tag,
                                 span,
                                 "href",
                                 sizeof("href") - 1U,
                                 &href_off,
                                 &href_len);
    ctx->active_link = priv_reflow_tok_intern_link(ctx->engine, href_off, href_len);
  }
  return k_ra8_ok;
}

RA8_INTERNAL
static void internal_handle_link(tok_ctx_t* ctx, const uint8_t* buf, size_t tag_lt, size_t end)
{
  if (ctx->engine->css_loader == nullptr) {
    return;
  }
  const uint8_t* tagbuf   = &buf[tag_lt];
  const size_t   span     = (end > tag_lt) ? (end - tag_lt) : 0U;
  size_t         rel_off  = 0U;
  size_t         rel_len  = 0U;
  size_t         href_off = 0U;
  size_t         href_len = 0U;
  if (!priv_reflow_tok_find_attr(tagbuf, span, "rel", sizeof("rel") - 1U, &rel_off, &rel_len) ||
      !priv_reflow_tok_find_attr(tagbuf, span, "href", sizeof("href") - 1U, &href_off, &href_len) ||
      !priv_reflow_tok_rel_is_stylesheet(&tagbuf[rel_off], rel_len)) {
    return;
  }
  const uint8_t* css_bytes = nullptr;
  size_t         css_len   = 0U;
  if ((ctx->engine->css_loader(ctx->engine->css_loader_ctx,
                               (const char*)&tagbuf[href_off],
                               (uint32_t)href_len,
                               &css_bytes,
                               &css_len) == k_ra8_ok) &&
      (css_bytes != nullptr) && (css_len > 0U)) {
    (void)ra8_css_parse(&ctx->engine->css, (const char*)css_bytes, (uint32_t)css_len);
  }
}

RA8_INTERNAL
static bool internal_handle_void(tok_ctx_t*        ctx,
                                 const uint8_t*    buf,
                                 size_t            tag_lt,
                                 size_t            end,
                                 reflow_html_tag_t tag,
                                 ra8_err_t*        out_err)
{
  if (tag == k_reflow_tag_link) {
    internal_handle_link(ctx, buf, tag_lt, end); /* external stylesheet (no token) */
    *out_err = k_ra8_ok;
    return true;
  }
  const bool is_void =
    (tag == k_reflow_tag_br) || (tag == k_reflow_tag_hr) || (tag == k_reflow_tag_img);
  if (is_void && internal_suppressed(ctx)) {
    *out_err = k_ra8_ok; /* void tag inside a display:none subtree -> drop */
    return true;
  }
  if (tag == k_reflow_tag_br) {
    *out_err = priv_reflow_tok_emit(ctx->engine, k_reflow_tok_break, tag, ctx->style, 0U, 0U)
                 ? k_ra8_ok
                 : k_ra8_err_no_mem;
    return true;
  }
  if (tag == k_reflow_tag_hr) {
    *out_err = priv_reflow_tok_emit(ctx->engine, k_reflow_tok_rule, tag, ctx->style, 0U, 0U)
                 ? k_ra8_ok
                 : k_ra8_err_no_mem;
    return true;
  }
  if (tag == k_reflow_tag_img) {
    uint32_t     src_off = 0U;
    uint32_t     src_len = 0U;
    const size_t span    = (end > tag_lt) ? (end - tag_lt) : 0U;
    priv_reflow_tok_capture_attr(ctx->engine,
                                 &buf[tag_lt],
                                 span,
                                 "src",
                                 sizeof("src") - 1U,
                                 &src_off,
                                 &src_len);
    *out_err =
      priv_reflow_tok_emit(ctx->engine, k_reflow_tok_image, tag, ctx->style, src_off, src_len)
        ? k_ra8_ok
        : k_ra8_err_no_mem;
    return true;
  }
  return false;
}

RA8_INTERNAL
static uint16_t internal_css_font_px(const tok_ctx_t* ctx, const ra8_css_style_t* comp)
{
  if ((comp->set & (uint8_t)k_ra8_css_set_fontsize) == 0U) {
    return ctx->css_font_px; /* inherit (0 = none -> UA default) */
  }
  const uint16_t parent = (ctx->css_font_px != 0U) ? ctx->css_font_px : ctx->engine->font_px;
  uint32_t px = ((ra8_css_font_unit_t)comp->font_unit == k_ra8_css_font_pct)
                  ? (((uint32_t)parent * (uint32_t)comp->font_val) / (uint32_t)k_reflow_pct_full)
                  : (uint32_t)comp->font_val;
  if (px < (uint32_t)k_reflow_min_font_px) {
    px = (uint32_t)k_reflow_min_font_px;
  }
  if (px > (uint32_t)k_reflow_max_font_px) {
    px = (uint32_t)k_reflow_max_font_px;
  }
  return (uint16_t)px;
}

RA8_INTERNAL
static ra8_err_t internal_open_styled(tok_ctx_t*        ctx,
                                      const uint8_t*    tagbuf,
                                      size_t            span,
                                      reflow_html_tag_t tag,
                                      bool              block)
{
  const uint8_t           base = (uint8_t)(ctx->style | priv_reflow_tok_style_for(tag));
  const ra8_css_element_t el   = ctx->stack_el[ctx->sp - 1U]; /* pushed by the caller */
  ra8_css_style_t         inh  = {.set = (uint8_t)k_priv_style_mask, .style = base};
  if (ctx->color != (uint32_t)k_reflow_color_inherit) {
    inh.set   = (uint8_t)(inh.set | (uint8_t)k_ra8_css_set_color);
    inh.color = ctx->color;
  }
  /* font-family inherits (#109): seed the cascade with the parent's resolved
   * family so a child without its own `font-family` keeps the ancestor's face. */
  if (ctx->family_len != 0U) {
    inh.set        = (uint8_t)(inh.set | (uint8_t)k_ra8_css_set_family);
    inh.family_off = ctx->family_off;
    inh.family_len = ctx->family_len;
  }
  const ra8_css_style_t inl = priv_reflow_tok_css_inline(tagbuf, span);
  /* Ancestors = the open-element stack below this one (stack_el[0 .. sp-2]). */
  const uint8_t         n_anc = (uint8_t)(ctx->sp - 1U);
  const ra8_css_style_t comp =
    ra8_css_cascade_ctx(&ctx->engine->css, &el, inh, inl, ctx->stack_el, n_anc);
  const uint16_t fpx = internal_css_font_px(ctx, &comp);
  /* display:none (#140): begin suppressing this element + its subtree at the
   * current depth; internal_open_attrs and the emit sites then drop every token
   * until the matching close pops back above this depth. */
  const bool hidden = ((comp.set & (uint8_t)k_ra8_css_set_display) != 0U) && (comp.display != 0U);
  if (hidden && (ctx->suppress_sp == 0U)) {
    ctx->suppress_sp = ctx->sp;
  }
  const ra8_err_t aerr = internal_open_attrs(ctx, tagbuf, span, tag, block, &comp, fpx);
  if (aerr != k_ra8_ok) {
    return aerr;
  }
  ctx->style = (uint8_t)(comp.style & (uint8_t)k_priv_style_mask);
  ctx->color = ((comp.set & (uint8_t)k_ra8_css_set_color) != 0U) ? comp.color
                                                                 : (uint32_t)k_reflow_color_inherit;
  ctx->css_font_px = fpx;
  /* Resolve the embedded face for this element from its cascaded family +
   * emphasis (#109). Children inherit the family slice; text runs stamp the
   * resolved slot. Both are 0 / unchanged for content without `@font-face`. */
  if ((comp.set & (uint8_t)k_ra8_css_set_family) != 0U) {
    ctx->family_off = comp.family_off;
    ctx->family_len = comp.family_len;
  }
  ctx->face_slot = priv_reflow_tok_resolve_face_slot(ctx->engine, &comp);
  return k_ra8_ok;
}

RA8_INTERNAL
static ra8_err_t internal_handle_start(tok_ctx_t* ctx, const uint8_t* buf, size_t* pi, size_t len)
{
  const size_t            tag_lt    = *pi; /* index of '<' before parse */
  bool                    selfclose = false;
  const reflow_html_tag_t tag       = internal_parse_start(buf, pi, len, &selfclose);
  ctx->saw_element                  = true;
 
  ra8_err_t verr = k_ra8_ok;
  if (internal_handle_void(ctx, buf, tag_lt, *pi, tag, &verr)) {
    return verr;
  }
 
  const bool block = priv_reflow_tok_is_block(tag);
  if (selfclose) {
    if (block && !internal_suppressed(ctx)) {
      if (!priv_reflow_tok_emit(ctx->engine, k_reflow_tok_block_start, tag, ctx->style, 0U, 0U)) {
        return k_ra8_err_no_mem;
      }
      if (!priv_reflow_tok_emit(ctx->engine, k_reflow_tok_block_end, tag, ctx->style, 0U, 0U)) {
        return k_ra8_err_no_mem;
      }
    }
    return k_ra8_ok;
  }
 
  if (ctx->sp >= (uint32_t)k_priv_max_depth) {
    return k_ra8_err_no_mem;
  }
  const size_t span = (*pi > tag_lt) ? (*pi - tag_lt) : 0U;
  /* Record the element's full identity (tag + id + class, aliasing the chapter
   * buffer) so the cascade reads it back and descendant selectors can match it
   * as an ancestor of a later element. */
  ctx->stack_el[ctx->sp]      = priv_reflow_tok_css_element(tag, &buf[tag_lt], span);
  ctx->stack_style[ctx->sp]   = ctx->style;
  ctx->stack_link[ctx->sp]    = ctx->active_link;
  ctx->stack_color[ctx->sp]   = ctx->color;
  ctx->stack_font[ctx->sp]    = ctx->css_font_px;
  ctx->stack_fam_off[ctx->sp] = ctx->family_off;
  ctx->stack_fam_len[ctx->sp] = ctx->family_len;
  ctx->stack_face[ctx->sp]    = ctx->face_slot;
  ctx->sp++;
  return internal_open_styled(ctx, &buf[tag_lt], span, tag, block);
}

RA8_INTERNAL
static bool internal_tag_is(const uint8_t* buf, size_t i, size_t len, const char* name)
{
  if (!priv_reflow_tok_starts_with(buf, i, len, name)) {
    return false;
  }
  const size_t n = strlen(name);
  if ((i + n) >= len) {
    return true;
  }
  const char c = (char)buf[i + n];
  return (c == '>') || (c == '/') || priv_reflow_tok_is_xml_whitespace(c);
}

RA8_INTERNAL
static void internal_handle_raw_text(tok_ctx_t*     ctx,
                                     const uint8_t* buf,
                                     size_t*        pi,
                                     size_t         len,
                                     const char*    close_lit,
                                     bool           is_style)
{
  ctx->saw_element      = true;
  const size_t open_end = priv_reflow_tok_skip_past(buf, *pi, len, ">");
  const size_t close_at = priv_reflow_tok_find_lit(buf, open_end, len, close_lit);
  if (is_style && (close_at > open_end)) {
    (void)ra8_css_parse(&ctx->engine->css,
                        (const char*)&buf[open_end],
                        (uint32_t)(close_at - open_end));
  }
  *pi = (close_at < len) ? (close_at + strlen(close_lit)) : len;
}

RA8_INTERNAL
static ra8_err_t internal_handle_lt(tok_ctx_t* ctx, const uint8_t* buf, size_t* pi, size_t len)
{
  if (priv_reflow_tok_starts_with(buf, *pi, len, "<!--")) {
    *pi = priv_reflow_tok_skip_past(buf, *pi + 4U, len, "-->");
    return k_ra8_ok;
  }
  if (priv_reflow_tok_starts_with(buf, *pi, len, "<![CDATA[")) {
    return internal_handle_cdata(ctx, buf, pi, len);
  }
  if (priv_reflow_tok_starts_with(buf, *pi, len, "<?")) {
    *pi = priv_reflow_tok_skip_past(buf, *pi + 2U, len, "?>");
    return k_ra8_ok;
  }
  if (priv_reflow_tok_starts_with(buf, *pi, len, "<!")) {
    *pi = priv_reflow_tok_skip_past(buf, *pi + 2U, len, ">");
    return k_ra8_ok;
  }
  if (internal_tag_is(buf, *pi, len, "<style")) {
    internal_handle_raw_text(ctx, buf, pi, len, "</style>", true);
    return k_ra8_ok;
  }
  if (internal_tag_is(buf, *pi, len, "<script")) {
    internal_handle_raw_text(ctx, buf, pi, len, "</script>", false);
    return k_ra8_ok;
  }
  if (((*pi + 1U) < len) && (buf[*pi + 1U] == (uint8_t)'/')) {
    return internal_handle_end(ctx, buf, pi, len);
  }
  return internal_handle_start(ctx, buf, pi, len);
}

RA8_INTERNAL
static bool internal_emit_text_run(tok_ctx_t* ctx, const uint8_t* buf, size_t run, size_t end)
{
  if ((ctx->sp == 0U) || internal_suppressed(ctx)) {
    return true; /* outside any element, or inside display:none -> drop */
  }
  uint32_t off = 0U;
  uint32_t tln = 0U;
  if (!priv_reflow_tok_stash_run(ctx->engine, buf, run, end, &off, &tln)) {
    return false;
  }
  if (tln == 0U) {
    return true;
  }
  if (!priv_reflow_tok_emit(ctx->engine,
                            k_reflow_tok_text,
                            k_reflow_tag_unknown,
                            ctx->style,
                            off,
                            tln)) {
    return false;
  }
  ctx->engine->tokens[ctx->engine->token_count - 1U].reserved   = ctx->active_link;
  ctx->engine->tokens[ctx->engine->token_count - 1U].color      = ctx->color;
  ctx->engine->tokens[ctx->engine->token_count - 1U].reserved16 = (uint16_t)ctx->face_slot;
  return true;
}
 
/* ===========================================================================
 * Entry point (called by reflow_parse.c)
 * ===========================================================================
 */
 
ra8_err_t priv_reflow_xml_walk(reflow_t* engine, const uint8_t* xhtml_buf, size_t xhtml_len)
{
  if ((engine == nullptr) || (xhtml_buf == nullptr)) {
    return k_ra8_err_null_ptr;
  }
  if (xhtml_len == 0U) {
    return k_ra8_err_invalid_size;
  }
 
  (void)ra8_css_sheet_reset(&engine->css); /* fresh CSS rules per chapter (#111) */
 
  /* The tokenizer scratch (per-open-element cascade stacks) is ~2 kB, which
   * would push this frame past the project stack-usage budget. Hold it in
   * module-static storage instead of on the stack and reset it on entry.
   * Safe: priv_reflow_xml_walk has a single, non-recursive, single-threaded
   * caller (priv_reflow_parse), so the shared instance never overlaps. */
  static tok_ctx_t s_walk_ctx;
  s_walk_ctx     = (tok_ctx_t){};
  tok_ctx_t* ctx = &s_walk_ctx;
  ctx->engine    = engine;
  ctx->style     = (uint8_t)k_reflow_style_normal;
  ctx->color     = (uint32_t)k_reflow_color_inherit;
 
  size_t i = 0U;
  while (i < xhtml_len) {
    if (xhtml_buf[i] == (uint8_t)'<') {
      const ra8_err_t err = internal_handle_lt(ctx, xhtml_buf, &i, xhtml_len);
      if (err != k_ra8_ok) {
        return err;
      }
      continue;
    }
    const size_t run = i;
    while ((i < xhtml_len) && (xhtml_buf[i] != (uint8_t)'<')) {
      ++i;
    }
    if (!internal_emit_text_run(ctx, xhtml_buf, run, i)) {
      return k_ra8_err_no_mem;
    }
  }
 
  if (!ctx->saw_element) {
    return k_ra8_err_validation_failed;
  }
  if (ctx->sp != 0U) {
    return k_ra8_err_validation_failed;
  }
  return k_ra8_ok;
}