mdl_app_series.c

Go to the documentation of this file.

#include "mdl_app_internal.h"
#include "mdl_fetch_body_internal.h"

static mdl_url_list_t s_selected;

static mdl_fetch_faillog_t s_faillog;

RA8_INTERNAL static ra8_err_t internal_series_text3(ra8_io_stream_t* stream,
                                                    const char*      first,
                                                    const char*      second,
                                                    const char*      third)
{
  ra8_err_t error = priv_mdl_stream_text(k_ra8_ok, stream, first);
  error           = priv_mdl_stream_text(error, stream, second);
  return priv_mdl_stream_text(error, stream, third);
}

RA8_INTERNAL static bool internal_list_range(const mdl_url_list_t* l, double* lo, double* hi)
{
  *lo = 0.0;
  *hi = 0.0;
  if (l->count == 0U) {
    return false;
  }
  if (!mdl_urlname_chapter_parse(l->urls[0], lo)) {
    return false;
  }
  *hi = *lo;
  for (size_t i = 1U; i < l->count; ++i) {
    double num = 0.0;
    if (!mdl_urlname_chapter_parse(l->urls[i], &num)) {
      return false;
    }
    *lo = (num < *lo) ? num : *lo;
    *hi = (num > *hi) ? num : *hi;
  }
  return true;
}

RA8_INTERNAL static void
internal_select_window(bool from_present, double from_num, size_t count, mdl_url_list_t* out)
{
  size_t start = 0U;
  if (from_present) {
    while (start < priv_mdl_app_context()->chapters.count) {
      double value = 0.0;
      if (mdl_urlname_chapter_parse(priv_mdl_app_context()->chapters.urls[start], &value) &&
          (value >= from_num)) {
        break;
      }
      ++start;
    }
  }
  out->count = 0U;
  for (size_t i = start; (i < priv_mdl_app_context()->chapters.count) && (out->count < count);
       ++i) {
    memcpy(out->urls[out->count], priv_mdl_app_context()->chapters.urls[i], k_mdl_url_max);
    out->count += 1U;
  }
}

RA8_PRIV bool priv_mdl_app_state_path_of(const char* abs_dir, char* out, size_t cap)
{
  const int n = snprintf(out, cap, "%s/.mdl_state", abs_dir);
  if (n < 0) {
    return false;
  }
  return (size_t)n < cap;
}

RA8_INTERNAL static bool internal_copy_metadata_text(char* dst, size_t cap, const char* src)
{
  if ((dst == nullptr) || (cap == 0U) || (src == nullptr)) {
    return false;
  }
  const int n = snprintf(dst, cap, "%s", src);
  if ((n < 0) || ((size_t)n >= cap)) {
    dst[0] = '\0';
    return false;
  }
  return true;
}

RA8_INTERNAL static bool internal_build_export_metadata(const char*              abs_dir,
                                                        const mdl_chapter_rec_t* rec,
                                                        mdl_export_meta_t*       meta)
{
  mdl_meta_init(meta);
  if (!internal_copy_metadata_text(meta->series_title,
                                   sizeof(meta->series_title),
                                   priv_mdl_app_context()->state.series_title) ||
      !internal_copy_metadata_text(meta->summary,
                                   sizeof(meta->summary),
                                   priv_mdl_app_context()->state.summary) ||
      !internal_copy_metadata_text(meta->writer,
                                   sizeof(meta->writer),
                                   priv_mdl_app_context()->state.writer) ||
      !internal_copy_metadata_text(meta->artist,
                                   sizeof(meta->artist),
                                   priv_mdl_app_context()->state.artist) ||
      !internal_copy_metadata_text(meta->source_url,
                                   sizeof(meta->source_url),
                                   priv_mdl_app_context()->state.series_url) ||
      !internal_copy_metadata_text(meta->language,
                                   sizeof(meta->language),
                                   priv_mdl_app_context()->state.language)) {
    return false;
  }
  meta->reading_direction =
    (priv_mdl_app_context()->state.reading_direction == k_mdl_state_read_rtl) ? k_mdl_read_rtl
                                                                              : k_mdl_read_ltr;
  if (priv_mdl_app_context()->state.cover_path[0] != '\0') {
    if (!mdl_path_join(abs_dir,
                       priv_mdl_app_context()->state.cover_path,
                       meta->cover_path,
                       sizeof(meta->cover_path))) {
      return false;
    }
  }
  uint64_t identity = mdl_hash_str(priv_mdl_app_context()->state.series_url);
  if (rec != nullptr) {
    if (!internal_copy_metadata_text(meta->chapter_title,
                                     sizeof(meta->chapter_title),
                                     rec->title)) {
      return false;
    }
    meta->chapter_number = rec->number_known ? rec->number : 0.0;
    identity             = mdl_hash_bytes_seed(rec->source_url, strlen(rec->source_url), identity);
  }
  const int id_len =
    snprintf(meta->identifier, sizeof(meta->identifier), "urn:media-dl:%016" PRIx64, identity);
  return (id_len >= 0) && ((size_t)id_len < sizeof(meta->identifier));
}

RA8_INTERNAL static size_t internal_export_fresh_separate(mdl_format_t          format,
                                                          const char*           abs_dir,
                                                          const mdl_url_list_t* sel,
                                                          int64_t               run_start)
{
  size_t fails = 0U;
  for (size_t i = 0U; i < sel->count; ++i) {
    char id[k_mdl_chapter_id_max];
    mdl_urlname_last_segment(sel->urls[i], id, sizeof(id));
    const mdl_chapter_rec_t* rec = mdl_state_find_chapter(&priv_mdl_app_context()->state, id);
    if ((rec != nullptr) && rec->complete && (rec->fetched_at >= run_start)) {
      mdl_export_meta_t meta;
      if (!internal_build_export_metadata(abs_dir, rec, &meta)) {
        (void)internal_series_text3(priv_mdl_app_context()->diagnostic,
                                    "  export metadata for ",
                                    id,
                                    " exceeded a bounded field\n");
        fails += 1U;
      } else {
        fails += mdl_pack_one_meta(&priv_mdl_app_context()->storage,
                                   format,
                                   abs_dir,
                                   id,
                                   &meta,
                                   &priv_mdl_app_context()->export_ws,
                                   priv_mdl_app_context()->output,
                                   priv_mdl_app_context()->diagnostic);
      }
    }
  }
  return fails;
}

RA8_INTERNAL static ra8_err_t internal_report_stats(const char*              abs_dir,
                                                    const mdl_fetch_stats_t* st)
{
  ra8_io_stream_t* output = priv_mdl_app_context()->output;
  ra8_err_t        error  = internal_series_text3(output, "done into ", abs_dir, "/: ");
  error                   = priv_mdl_stream_u64(error, output, st->chapters_completed);
  error                   = priv_mdl_stream_text(error, output, " chapter(s) fetched, ");
  error                   = priv_mdl_stream_u64(error, output, st->chapters_skipped);
  error                   = priv_mdl_stream_text(error, output, " skipped, ");
  error                   = priv_mdl_stream_u64(error, output, st->chapters_failed);
  error                   = priv_mdl_stream_text(error, output, " failed; pages ");
  error                   = priv_mdl_stream_u64(error, output, st->pages_fetched);
  error                   = priv_mdl_stream_text(error, output, " new / ");
  error                   = priv_mdl_stream_u64(error, output, st->pages_reused);
  error                   = priv_mdl_stream_text(error, output, " reused / ");
  error                   = priv_mdl_stream_u64(error, output, st->pages_failed);
  return priv_mdl_stream_text(error, output, " failed\n");
}

RA8_INTERNAL static mdl_fetch_layout_t internal_choose_layout(const mdl_series_run_t* r,
                                                              const mdl_url_list_t*   sel,
                                                              char*                   combined_rel,
                                                              size_t                  cap,
                                                              const char*             slug)
{
  combined_rel[0] = '\0';
  if (r->update || !r->combine || (r->format == k_mdl_format_loose)) {
    return k_mdl_layout_separate;
  }
  double lo = 0.0;
  double hi = 0.0;
  if (internal_list_range(sel, &lo, &hi)) {
    (void)snprintf(combined_rel, cap, "%s-%.10g-%.10g", slug, lo, hi);
  } else {
    (void)snprintf(combined_rel, cap, "%s-%zu-chapters", slug, sel->count);
  }
  return k_mdl_layout_combined;
}

RA8_INTERNAL static mdl_fetch_ctx_t internal_make_ctx(const mdl_series_run_t* r,
                                                      const mdl_site_t*       site,
                                                      const char*             abs_dir,
                                                      const char*             state_path,
                                                      mdl_governor_t*         gov,
                                                      mdl_cache_t*            cache)
{
  return (mdl_fetch_ctx_t){.session        = &priv_mdl_app_context()->session,
                           .storage        = &priv_mdl_app_context()->storage,
                           .state          = &priv_mdl_app_context()->state,
                           .cache          = cache,
                           .state_path     = state_path,
                           .series_abs_dir = abs_dir,
                           .series_url     = r->series_url,
                           .site           = site,
                           .gov            = gov,
                           .timeout_ms     = r->timeout,
                           .page_buf       = priv_mdl_app_context()->page,
                           .page_cap       = sizeof(priv_mdl_app_context()->page),
                           .images         = &priv_mdl_app_context()->images,
                           .update_only    = r->update,
                           .refetch        = r->opts->refetch,
                           .faillog        = &s_faillog,
                           .diagnostic     = priv_mdl_app_context()->diagnostic,
                           .progress_fn =
                             r->opts->progress ? mdl_report_progress_bar : mdl_report_progress,
                           .progress_ctx   = priv_mdl_app_context()->output,
                           .progress_error = k_ra8_ok};
}

RA8_INTERNAL static size_t internal_export_after(const mdl_series_run_t*  r,
                                                 const char*              abs_dir,
                                                 mdl_fetch_layout_t       layout,
                                                 const char*              combined_rel,
                                                 const mdl_url_list_t*    sel,
                                                 const mdl_fetch_stats_t* stats,
                                                 int64_t                  run_start)
{
  if (r->format == k_mdl_format_loose) {
    return 0U;
  }
  if (layout == k_mdl_layout_combined) {
    mdl_export_meta_t meta;
    if (!internal_build_export_metadata(abs_dir, nullptr, &meta)) {
      (void)priv_mdl_stream_text(k_ra8_ok,
                                 priv_mdl_app_context()->diagnostic,
                                 "  combined export metadata exceeded a bounded field\n");
      return 1U;
    }
    return mdl_pack_combined_meta(&priv_mdl_app_context()->storage,
                                  r->format,
                                  r->opts->allow_incomplete,
                                  abs_dir,
                                  combined_rel,
                                  stats,
                                  &meta,
                                  &priv_mdl_app_context()->export_ws,
                                  priv_mdl_app_context()->output,
                                  priv_mdl_app_context()->diagnostic);
  }
  return internal_export_fresh_separate(r->format, abs_dir, sel, run_start);
}

RA8_INTERNAL static ra8_err_t internal_save_series_state(const char* state_path,
                                                         const char* failure)
{
  bool            published = false;
  const ra8_err_t error     = mdl_state_save(&priv_mdl_app_context()->storage,
                                             state_path,
                                             &priv_mdl_app_context()->state,
                                             &published);
  if (error != k_ra8_ok) {
    (void)internal_series_text3(priv_mdl_app_context()->diagnostic, failure, state_path, "'\n");
  }
  return error;
}

RA8_INTERNAL static void internal_report_cover_failure(ra8_err_t error)
{
  ra8_err_t output_error = priv_mdl_stream_text(k_ra8_ok,
                                                priv_mdl_app_context()->diagnostic,
                                                "mdl: series cover acquisition failed "
                                                "(err 0x");
  output_error =
    priv_mdl_stream_hex(output_error, priv_mdl_app_context()->diagnostic, (uint32_t)error);
  (void)priv_mdl_stream_text(output_error, priv_mdl_app_context()->diagnostic, ")\n");
}

RA8_INTERNAL static ra8_err_t internal_select_run_window(const mdl_series_run_t* r,
                                                         const mdl_url_list_t**  out_sel)
{
  *out_sel = &priv_mdl_app_context()->chapters;
  if (!r->update) {
    internal_select_window(r->from_present, r->from_num, r->chapters, &s_selected);
    *out_sel = &s_selected;
    if ((*out_sel)->count == 0U) {
      (void)priv_mdl_stream_text(k_ra8_ok,
                                 priv_mdl_app_context()->diagnostic,
                                 "mdl: no chapters match the requested range\n");
      return k_ra8_err_invalid_arg;
    }
  }
  return k_ra8_ok;
}

RA8_INTERNAL static int internal_run_prepared(const mdl_series_run_t* r,
                                              const mdl_site_t*       site,
                                              const char*             abs_dir,
                                              const char*             state_path,
                                              mdl_cache_t*            cache)
{
  const mdl_url_list_t* sel = nullptr;
  if (internal_select_run_window(r, &sel) != k_ra8_ok) {
    return 1;
  }
  char display_slug[k_slug_bytes];
  (void)mdl_sanitize_segment(priv_mdl_app_context()->state.series_title,
                             display_slug,
                             sizeof(display_slug));
  char               combined_rel[k_leaf_name_bytes];
  mdl_fetch_layout_t layout =
    internal_choose_layout(r, sel, combined_rel, sizeof(combined_rel), display_slug);
 
  mdl_governor_t      gov;
  const mdl_gov_cfg_t cfg = mdl_config_gov_cfg(site);
  mdl_governor_init(&gov, &cfg, r->seed);
  memset(&s_faillog, 0, sizeof(s_faillog));
  mdl_fetch_ctx_t ctx = internal_make_ctx(r, site, abs_dir, state_path, &gov, cache);
  const ra8_err_t crc = priv_mdl_app_ensure_series_cover(&ctx, abs_dir);
  if (crc != k_ra8_ok) {
    internal_report_cover_failure(crc);
    return 1;
  }
  if (internal_save_series_state(state_path, "mdl: failed to checkpoint series metadata '") !=
      k_ra8_ok) {
    return 1;
  }
  const int64_t     run_start = (int64_t)time(nullptr);
  mdl_fetch_stats_t stats;
  const ra8_err_t   frc = mdl_fetch_run(&ctx,
                                        sel,
                                        layout,
                                        (layout == k_mdl_layout_combined) ? combined_rel : nullptr,
                                        &stats);
  if (ctx.progress_error != k_ra8_ok) {
    return 1;
  }
  const ra8_err_t src =
    internal_save_series_state(state_path, "mdl: failed to save library state '");
  if (internal_report_stats(abs_dir, &stats) != k_ra8_ok) {
    return 1;
  }
  if (mdl_report_failures(priv_mdl_app_context()->diagnostic, &s_faillog) != k_ra8_ok) {
    return 1;
  }
  const size_t efail =
    internal_export_after(r, abs_dir, layout, combined_rel, sel, &stats, run_start);
  return ((frc == k_ra8_ok) && (src == k_ra8_ok) && (efail == 0U)) ? 0 : 1;
}

typedef struct {
  char title[k_mdl_title_max];        
  char summary[k_mdl_summary_max];    
  char writer[k_mdl_person_max];      
  char artist[k_mdl_person_max];      
  char cover_path[k_mdl_relpath_max]; 
} series_prior_metadata_t;

RA8_INTERNAL static void internal_snapshot_prior_metadata(series_prior_metadata_t* prior)
{
  const mdl_app_context_t* app = priv_mdl_app_context();
  (void)snprintf(prior->title, sizeof(prior->title), "%s", app->state.series_title);
  (void)snprintf(prior->summary, sizeof(prior->summary), "%s", app->state.summary);
  (void)snprintf(prior->writer, sizeof(prior->writer), "%s", app->state.writer);
  (void)snprintf(prior->artist, sizeof(prior->artist), "%s", app->state.artist);
  (void)snprintf(prior->cover_path,
                 sizeof(prior->cover_path),
                 "%s",
                 (strcmp(app->state.cover_url, app->series_metadata.cover_url) == 0)
                   ? app->state.cover_path
                   : "");
}

RA8_INTERNAL static bool internal_reconcile_series_state(const mdl_series_run_t*        run,
                                                         const mdl_site_t*              site,
                                                         const series_prior_metadata_t* prior)
{
  mdl_app_context_t* app   = priv_mdl_app_context();
  const char*        title = app->series_metadata.title;
  if (!app->series_metadata.title_selected && (prior->title[0] != '\0')) {
    title = prior->title;
    if (priv_mdl_stream_text(k_ra8_ok,
                             priv_mdl_app_context()->diagnostic,
                             "mdl: WARNING: preserving recorded series title after "
                             "selector failure\n") != k_ra8_ok) {
      return false;
    }
  }
  const char* summary =
    (app->series_metadata.summary[0] != '\0') ? app->series_metadata.summary : prior->summary;
  const char* writer =
    (app->series_metadata.writer[0] != '\0') ? app->series_metadata.writer : prior->writer;
  const char* artist =
    (app->series_metadata.artist[0] != '\0') ? app->series_metadata.artist : prior->artist;
  mdl_state_set_series(&app->state, run->series_url, title, site->name, site->host, run->cfg_path);
  return mdl_state_set_series_metadata(&app->state,
                                       summary,
                                       writer,
                                       artist,
                                       app->series_metadata.cover_url,
                                       prior->cover_path,
                                       app->series_metadata.language,
                                       app->series_metadata.direction);
}

RA8_INTERNAL static int
internal_run_series_paths(const mdl_series_run_t* run, const mdl_site_t* site, mdl_cache_t* cache)
{
  char slug[k_slug_bytes];
  char abs_dir[k_fw_fs_path_cap];
  char state_path[k_fw_fs_path_cap];
  if (!priv_mdl_app_prepare_series_dir(run->out_dir,
                                       run->series_url,
                                       slug,
                                       sizeof(slug),
                                       abs_dir) ||
      !priv_mdl_app_state_path_of(abs_dir, state_path, sizeof(state_path))) {
    return 1;
  }
  const ra8_err_t load_rc =
    mdl_state_load(&priv_mdl_app_context()->storage, state_path, &priv_mdl_app_context()->state);
  if (load_rc != k_ra8_ok) {
    ra8_err_t output_error = internal_series_text3(priv_mdl_app_context()->diagnostic,
                                                   "mdl: refusing to overwrite unreadable "
                                                   "state '",
                                                   state_path,
                                                   "' (err 0x");
    output_error =
      priv_mdl_stream_hex(output_error, priv_mdl_app_context()->diagnostic, (uint32_t)load_rc);
    (void)priv_mdl_stream_text(output_error, priv_mdl_app_context()->diagnostic, ")\n");
    return 1;
  }
  series_prior_metadata_t prior;
  internal_snapshot_prior_metadata(&prior);
  if (!internal_reconcile_series_state(run, site, &prior)) {
    (void)priv_mdl_stream_text(k_ra8_ok,
                               priv_mdl_app_context()->diagnostic,
                               "mdl: scraped series metadata exceeded state bounds\n");
    return 1;
  }
  return internal_run_prepared(run, site, abs_dir, state_path, cache);
}

RA8_INTERNAL static int internal_run_series_network(const mdl_series_run_t* run, mdl_site_t* site)
{
  mdl_net_iface_t net = {};
  if (mdl_net_provider_open(run->opts->net, &run->opts->policy, &net) != k_ra8_ok) {
    (void)priv_mdl_stream_text(k_ra8_ok,
                               priv_mdl_app_context()->diagnostic,
                               "mdl: network init failed\n");
    return 1;
  }
  char ua[k_mdl_ua_max];
  if (priv_mdl_app_start_session(&net, run->opts, site->contact, ua, sizeof(ua)) != k_ra8_ok) {
    mdl_net_destroy(&net);
    return 1;
  }
 
  mdl_cache_t cache = {.storage    = &priv_mdl_app_context()->storage,
                       .index      = &priv_mdl_app_context()->cache_index,
                       .root       = run->cache_dir,
                       .diagnostic = priv_mdl_app_context()->diagnostic,
                       .refetch    = run->opts->refetch};
  int         rc    = 1;
  if (priv_mdl_app_prepare_chapters(site, run->series_url, run->timeout, &cache) != k_ra8_ok) {
    (void)priv_mdl_stream_text(k_ra8_ok,
                               priv_mdl_app_context()->diagnostic,
                               "mdl: series fetch failed\n");
  } else if (priv_mdl_app_context()->chapters.count == 0U) {
    (void)priv_mdl_stream_text(k_ra8_ok,
                               priv_mdl_app_context()->diagnostic,
                               "mdl: no chapters (check chapter_url_contains)\n");
  } else {
    ra8_err_t output_error =
      priv_mdl_stream_text(k_ra8_ok, priv_mdl_app_context()->output, "chapters found: ");
    output_error = priv_mdl_stream_u64(output_error,
                                       priv_mdl_app_context()->output,
                                       priv_mdl_app_context()->chapters.count);
    output_error = priv_mdl_stream_text(output_error, priv_mdl_app_context()->output, "\n");
    if (output_error == k_ra8_ok) {
      rc = internal_run_series_paths(run, site, &cache);
    }
  }
  mdl_net_destroy(&net);
  return rc;
}
 
int mdl_app_run_series(const mdl_series_run_t* r)
{
  if ((r == nullptr) || (r->cache_dir == nullptr) ||
      (priv_mdl_app_storage_ensure_directory(&priv_mdl_app_context()->storage, r->out_dir) !=
       k_ra8_ok)) {
    return 1;
  }
  mdl_site_t site;
  if (mdl_config_load(&priv_mdl_app_context()->storage, r->cfg_path, &site) != k_ra8_ok) {
    return 1;
  }
  if (r->opts->polite) {
    mdl_config_apply_polite(&site);
  }
  ra8_err_t output_error =
    internal_series_text3(priv_mdl_app_context()->output, "site: ", site.name, " (host ");
  output_error = priv_mdl_stream_text(output_error, priv_mdl_app_context()->output, site.host);
  output_error = priv_mdl_stream_text(output_error, priv_mdl_app_context()->output, ", kind ");
  output_error = priv_mdl_stream_text(output_error, priv_mdl_app_context()->output, site.kind);
  output_error = priv_mdl_stream_text(output_error, priv_mdl_app_context()->output, ")\n");
  if (output_error != k_ra8_ok) {
    return 1;
  }
 
  return internal_run_series_network(r, &site);
}