mdl_fetch.c File Reference

State-aware chapter, asset, and run download orchestration. More...

#include "mdl_fetch.h"
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include "mdl_fetch_body_internal.h"
#include "mdl_fetch_internal.h"
#include "mdl_net.h"
#include "mdl_pathfs.h"
#include "mdl_storage.h"
#include "mdl_stream_internal.h"
#include "mdl_url_guard.h"
#include "mdl_urlname.h"
#include "ra8_attributes.h"
#include "ra8_err.h"

Include dependency graph for mdl_fetch.c:

Go to the source code of this file.

Data Structures
struct	mdl_chapter_work_t
	Bounded metadata and state accumulated while processing one chapter. More...

Enumerations
enum	mdl_fetch_http_t : uint16_t { k_http_server_err = 500 }
	HTTP-status boundary used when rendering a failure reason. More...
enum	mdl_fetch_retry_t : uint8_t { k_fetch_max_attempts = 4U }
	Bounded attempts per request: the initial try plus backoff retries. More...
enum	mdl_chap_status_t : uint8_t {   k_ch_completed = 0 ,   k_ch_skipped = 1 ,   k_ch_failed = 2 }
	Per-chapter outcome reported to the run loop. More...

Functions
static ra8_err_t	internal_mdl_fetch_diag3 (mdl_fetch_ctx_t *ctx, const char *first, const char *second, const char *third)
	Emit three fetch diagnostics and latch a sink failure.
static uint32_t	internal_mdl_fetch_max_u32 (uint32_t a, uint32_t b)
	Perform the max u32 step.
bool	priv_mdl_fetch_is_retryable (ra8_err_t rc)
	Whether a failed transfer result is worth retrying.
bool	priv_mdl_fetch_run_incomplete (const mdl_fetch_stats_t *stats)
	Whether a finished run left anything unfetched (chapter or page).
static const char *	internal_mdl_fetch_fail_reason (long status)
	Prose for a k_ra8_fail: a 5xx server error versus a transport error.
void	priv_mdl_fetch_reason (ra8_err_t err, long status, char *buf, size_t cap)
	Render a human-readable reason for a transfer result and HTTP status.
void	priv_mdl_fetch_record_fail (const mdl_fetch_ctx_t *ctx, const char *url, long status, ra8_err_t err)
	Append one failure to the run's log (when set); always tally total.
ra8_err_t	priv_mdl_fetch_checkpoint (const mdl_fetch_ctx_t *ctx)
	Persist state atomically when a checkpoint target is configured.
static const char *	internal_mdl_fetch_page_host (const char *url, char *buf, size_t cap)
	Governor host key for url, or NULL when it cannot be parsed.
static ra8_err_t	internal_mdl_fetch_governed_get_body (mdl_fetch_ctx_t *ctx, const char *host, const char *url, const mdl_net_req_t *req, mdl_net_body_sink_t *sink, uint32_t jmin, uint32_t jmax, mdl_net_resp_t *out_resp, size_t *out_bytes)
	One governed body transfer: pace, fetch, feed the outcome back.
ra8_err_t	priv_mdl_fetch_with_retry (mdl_fetch_ctx_t *ctx, const char *host, const char *url, const mdl_net_req_t *req, mdl_net_body_sink_t *sink, uint32_t jmin, uint32_t jmax, mdl_net_resp_t *out_resp, size_t *out_bytes)
	Bounded, governed retry of one image transfer; last status latched.
static ra8_err_t	internal_fetch_asset_execute (mdl_fetch_ctx_t *ctx, const char *host, const char *url, const mdl_net_req_t *req, uint32_t jmin, uint32_t jmax, const char *target_abs, mdl_net_resp_t *out_resp, size_t *out_bytes)
	Fetch one asset body and commit it to permanent storage.
ra8_err_t	mdl_fetch_asset (mdl_fetch_ctx_t *ctx, const char *url, const char *target_abs, const char *referer, mdl_net_resp_t *out_resp, size_t *out_bytes)
	Fetch one policy-governed asset to an absolute file path.
ra8_err_t	priv_mdl_fetch_cache_get_buf (void *context, const char *url, const mdl_net_req_t *request, char *buffer, size_t capacity, size_t *out_length, mdl_net_resp_t *response)
	Fetch one cache attempt through governor and bounded retry policy.
static ra8_err_t	internal_mdl_fetch_chapter_html (mdl_fetch_ctx_t *ctx, const char *chapter_url, size_t *out_len)
	Fetch one chapter HTML document and extract its page URLs.
static bool	internal_mdl_fetch_resolve_dest (mdl_fetch_ctx_t *ctx, mdl_fetch_layout_t layout, const char *id, const char *combined_abs, const char *combined_rel, size_t global_no, char *chap_abs, size_t chap_cap, const char **dest_abs, const char **dest_rel, size_t *base)
	Resolve the output directory and starting page number for one chapter.
static void	internal_mdl_fetch_mark_complete (mdl_fetch_ctx_t *ctx, mdl_fetch_layout_t layout, mdl_chapter_rec_t *rec, size_t *global_no)
	Mark a chapter complete and advance combined numbering.
static bool	internal_mdl_fetch_select_chapter_title (mdl_fetch_ctx_t *ctx, const char *chapter_url, mdl_chapter_work_t *work)
	Select one bounded persistent chapter title.
static bool	internal_mdl_fetch_select_chapter_number (mdl_fetch_ctx_t *ctx, const char *chapter_url, mdl_chapter_work_t *work)
	Select and strictly parse optional descriptor-driven chapter numbering.
static mdl_chap_status_t	internal_mdl_fetch_chapter_pages_and_checkpoint (mdl_fetch_ctx_t *ctx, const char *chapter_url, mdl_fetch_layout_t layout, const char *combined_abs, const char *combined_rel, size_t *global_no, mdl_fetch_stats_t *stats, size_t chapter_index, size_t chapter_total, mdl_chapter_work_t *work)
	Persist metadata, fetch pages, and checkpoint one prepared chapter.
static mdl_chap_status_t	internal_mdl_fetch_process_chapter (mdl_fetch_ctx_t *ctx, const char *chapter_url, mdl_fetch_layout_t layout, const char *combined_abs, const char *combined_rel, size_t *global_no, mdl_fetch_stats_t *stats, size_t chapter_index, size_t chapter_total)
	Download one chapter (or skip it), returning its outcome.
static bool	internal_mdl_fetch_tally (mdl_chap_status_t s, mdl_fetch_stats_t *stats)
	Fold one chapter outcome into run statistics.
static bool	internal_mdl_fetch_ctx_ready (const mdl_fetch_ctx_t *ctx)
	Validate that every required fetch dependency is present.
ra8_err_t	mdl_fetch_run (mdl_fetch_ctx_t *ctx, const mdl_url_list_t *chapters, mdl_fetch_layout_t layout, const char *combined_dir_rel, mdl_fetch_stats_t *stats)
	Download a series' chapters incrementally, resuming and deduping.

Detailed Description

State-aware chapter, asset, and run download orchestration.

Coordinates injected network, storage, verification, checkpointing, and progress contracts while preserving the first failure.

Copyright

Copyright (c) 2026 Brighton Sikarskie SPDX-License-Identifier: MIT

Definition in file mdl_fetch.c.

Enumeration Type Documentation

mdl_chap_status_t

enum mdl_chap_status_t : uint8_t

Per-chapter outcome reported to the run loop.

Enumerator
k_ch_completed	Every page present and verified.
k_ch_skipped	Already complete (–update skip).
k_ch_failed	Left partial by a page failure.

Definition at line 39 of file mdl_fetch.c.

mdl_fetch_http_t

enum mdl_fetch_http_t : uint16_t

HTTP-status boundary used when rendering a failure reason.

Enumerator
k_http_server_err	First status that is a server error.

Definition at line 29 of file mdl_fetch.c.

mdl_fetch_retry_t

enum mdl_fetch_retry_t : uint8_t

Bounded attempts per request: the initial try plus backoff retries.

Enumerator
k_fetch_max_attempts	One initial request plus at most three retries.

Definition at line 34 of file mdl_fetch.c.

Function Documentation

internal_fetch_asset_execute()

ra8_err_t internal_fetch_asset_execute ( mdl_fetch_ctx_t * ctx, const char * host, const char * url, const mdl_net_req_t * req, uint32_t jmin, uint32_t jmax, const char * target_abs, mdl_net_resp_t * out_resp, size_t * out_bytes )

static

Fetch one asset body and commit it to permanent storage.

Runs the retry-governed GET, publishes the response header to the caller, then prepares and commits (or aborts) the streamed body.

Parameters

[in,out]	ctx	Active fetch session and site context.
[in]	host	Resolved request host.
[in]	url	Absolute or site-relative asset URL.
[in]	req	Prepared network request (referer, user agent, timeout).
[in]	jmin	Minimum inter-request jitter in milliseconds.
[in]	jmax	Maximum inter-request jitter in milliseconds.
[in]	target_abs	Canonical destination path.
[out]	out_resp	Response header, published even on failure.
[out]	out_bytes	Bytes committed on success.

Returns

Fetch/commit status.

Return values

k_ra8_ok	The body was fetched and committed.
other	The fetch, prepare, or commit stage failed (already aborted and recorded).

Precondition

host, url, and target_abs are validated by the caller.

ctx holds an initialized storage binding and failure log.

Postcondition

On failure the transaction is aborted and the failure is recorded.

On success the body transaction is committed exactly once and out_bytes, when non-NULL, holds its length.

Note

Not thread-safe with respect to concurrent callers sharing ctx.

Since

0.1.0

Definition at line 317 of file mdl_fetch.c.

References k_ra8_ok, priv_mdl_fetch_body_abort(), priv_mdl_fetch_body_commit(), priv_mdl_fetch_body_init_exact(), priv_mdl_fetch_body_prepare(), priv_mdl_fetch_body_sink(), priv_mdl_fetch_record_fail(), priv_mdl_fetch_with_retry(), RA8_INTERNAL, mdl_net_resp_t::status, and mdl_fetch_ctx_t::storage.

Referenced by mdl_fetch_asset().

internal_mdl_fetch_chapter_html()

ra8_err_t internal_mdl_fetch_chapter_html ( mdl_fetch_ctx_t * ctx, const char * chapter_url, size_t * out_len )

static

Fetch one chapter HTML document and extract its page URLs.

Applies robots policy, governed bounded retries, then retains the successful HTML bytes in ctx->page_buf for immediate metadata use.

Parameters

[in,out]	ctx	Fully configured fetch context.
[in]	chapter_url	Absolute chapter URL.
[out]	out_len	Retained HTML byte count.

Returns

An ra8_err_t fetch/extraction result.

Return values

k_ra8_ok	At least one page URL was extracted.
k_ra8_err_no_data	The HTML held no matching page image.
k_ra8_fail	Robots or network policy refused/failed the fetch.

Precondition

All pointer arguments are non-NULL.

ctx->page_buf and ctx->images are writable caller storage.

Postcondition

On success, out_len is nonzero and the HTML remains in the buffer.

On success, ctx->images->count is nonzero.

Note

Not thread-safe: mutates session, governor, and scratch storage.

Since

0.1.0

Definition at line 470 of file mdl_fetch.c.

References mdl_fetch_ctx_t::cache, mdl_site_t::chapter_delay_max, mdl_site_t::chapter_delay_min, mdl_url_list_t::count, mdl_fetch_cache_request::host, mdl_fetch_ctx_t::images, internal_mdl_fetch_max_u32(), k_ra8_err_invalid_arg, k_ra8_err_no_data, k_ra8_fail, k_ra8_ok, mdl_cache_get_buf(), mdl_extract_images(), mdl_session_url_allowed(), mdl_url_host(), mdl_fetch_ctx_t::page_buf, mdl_fetch_ctx_t::page_cap, mdl_site_t::page_img_attr, mdl_site_t::page_img_url_contains, priv_mdl_fetch_cache_get_buf(), priv_mdl_fetch_record_fail(), mdl_fetch_ctx_t::series_url, mdl_fetch_ctx_t::session, mdl_fetch_ctx_t::site, mdl_net_resp_t::status, mdl_fetch_ctx_t::timeout_ms, and mdl_session_t::user_agent.

Referenced by internal_mdl_fetch_process_chapter().

internal_mdl_fetch_chapter_pages_and_checkpoint()

mdl_chap_status_t internal_mdl_fetch_chapter_pages_and_checkpoint ( mdl_fetch_ctx_t * ctx, const char * chapter_url, mdl_fetch_layout_t layout, const char * combined_abs, const char * combined_rel, size_t * global_no, mdl_fetch_stats_t * stats, size_t chapter_index, size_t chapter_total, mdl_chapter_work_t * work )

static

Persist metadata, fetch pages, and checkpoint one prepared chapter.

Performs chapter pages and checkpoint under the injected network, governor, and storage contracts; dependency failures are propagated before incomplete bytes are published.

Parameters

[in,out]	ctx	Fetch dependencies and state.
[in]	chapter_url	Absolute chapter URL.
[in]	layout	Requested output layout.
[in]	combined_abs	Combined absolute output directory.
[in]	combined_rel	Combined relative output directory.
[in,out]	global_no	Combined page counter.
[in,out]	stats	Run counters.
[in]	chapter_index	One-based chapter position.
[in]	chapter_total	Total chapters in the run.
[in,out]	work	Prepared chapter metadata.

Returns

Completed or failed chapter status.

Return values

other

The computed result in the function's declared domain.

Precondition

Required pointer arguments remain valid for the call duration.

Supplied capacities cover their referenced bounded buffers.

Postcondition

No ownership of caller-provided storage is transferred.

Result status and outputs describe one completed synchronous attempt.

Note

The function performs no dynamic allocation and retains no caller pointer.

Since

0.1.0

Definition at line 770 of file mdl_fetch.c.

References mdl_url_list_t::count, mdl_fetch_ctx_t::images, internal_mdl_fetch_mark_complete(), internal_mdl_fetch_resolve_dest(), k_ch_completed, k_ch_failed, k_ra8_ok, mdl_state_set_chapter_metadata(), priv_mdl_fetch_chapter_pages(), and priv_mdl_fetch_checkpoint().

Referenced by internal_mdl_fetch_process_chapter().

internal_mdl_fetch_ctx_ready()

bool internal_mdl_fetch_ctx_ready ( const mdl_fetch_ctx_t * ctx )

static

Validate that every required fetch dependency is present.

Checks the network session, portable storage binding, persistent state, series/site metadata, and caller-owned extraction buffers before the orchestrator performs any I/O.

Parameters

[in]

ctx

Candidate fetch dependency bundle.

Returns

Dependency validation result.

Return values

true	Every mandatory pointer is non-null.
false	The context or at least one mandatory dependency is absent.

Precondition

ctx is either null or points to a readable context object.

No concurrent thread mutates a non-null context during this check.

Postcondition

No context, backend, or caller buffer is modified.

Success authorizes only pointer presence; callee operations still validate their detailed lifecycle and capacity contracts.

Note

Thread-safe for an immutable context.

Since

0.1.0

Definition at line 937 of file mdl_fetch.c.

References mdl_fetch_ctx_t::images, mdl_fetch_ctx_t::page_buf, RA8_INTERNAL, mdl_fetch_ctx_t::series_abs_dir, mdl_fetch_ctx_t::series_url, mdl_fetch_ctx_t::session, mdl_fetch_ctx_t::site, mdl_fetch_ctx_t::state, and mdl_fetch_ctx_t::storage.

Referenced by mdl_fetch_run().

internal_mdl_fetch_diag3()

ra8_err_t internal_mdl_fetch_diag3 ( mdl_fetch_ctx_t * ctx, const char * first, const char * second, const char * third )

static

Emit three fetch diagnostics and latch a sink failure.

Writes fragments in order through the injected diagnostic stream. The first sink error is latched in the fetch context and returned.

Parameters

[in,out]	ctx	Caller-owned operation context.
[in]	first	First text fragment.
[in]	second	Second text fragment.
[in]	third	Third text fragment.

Returns

Operation status.

Return values

k_ra8_ok	The operation completed successfully.
other	The originating validation, storage, stream, or network error.

Precondition

Every required pointer is non-null and remains valid for the call.

Lengths and capacities describe complete referenced objects without overflow.

Postcondition

Documented outputs and the return value describe the same outcome.

A rejected or failed operation is never reported as successful.

Note

Thread safety follows ownership of the supplied context; no synchronization is added.

Since

Version 0.1.0

Definition at line 77 of file mdl_fetch.c.

References mdl_fetch_ctx_t::diagnostic, k_ra8_ok, priv_mdl_stream_text(), mdl_fetch_ctx_t::progress_error, and RA8_INTERNAL.

Referenced by internal_mdl_fetch_select_chapter_number(), and internal_mdl_fetch_select_chapter_title().

internal_mdl_fetch_fail_reason()

const char * internal_mdl_fetch_fail_reason ( long status )

static

Prose for a k_ra8_fail: a 5xx server error versus a transport error.

Definition at line 125 of file mdl_fetch.c.

References k_http_server_err, and RA8_INTERNAL.

Referenced by priv_mdl_fetch_reason().

internal_mdl_fetch_governed_get_body()

ra8_err_t internal_mdl_fetch_governed_get_body ( mdl_fetch_ctx_t * ctx, const char * host, const char * url, const mdl_net_req_t * req, mdl_net_body_sink_t * sink, uint32_t jmin, uint32_t jmax, mdl_net_resp_t * out_resp, size_t * out_bytes )

static

One governed body transfer: pace, fetch, feed the outcome back.

Dispatches through an injected caller-owned sink and always releases the governor slot after the synchronous attempt.

Parameters

[in,out]	ctx	Borrowed operation context.
[in]	host	Origin host governing pacing and robots policy.
[in]	url	Canonical input URL.
[in]	req	Validated discovery or request description.
[in,out]	sink	Reset/write destination for this attempt.
[in]	jmin	Minimum retry-jitter delay in milliseconds.
[in]	jmax	Maximum retry-jitter delay in milliseconds.
[out]	out_resp	Receives normalized response metadata.
[out]	out_bytes	Receives the transferred byte count.

Returns

Canonical downloader status.

Return values

k_ra8_ok	The operation completed.
other	Validation, capacity, network, or storage failed.

Precondition

Required pointer arguments remain valid for the call duration.

Supplied capacities cover their referenced bounded buffers.

Postcondition

No ownership of caller-provided storage is transferred.

Result status and outputs describe one completed synchronous attempt.

Note

The function performs no dynamic allocation and retains no caller pointer.

Since

0.1.0

Definition at line 235 of file mdl_fetch.c.

References mdl_fetch_ctx_t::gov, k_ra8_err_would_block, mdl_governor_acquire(), mdl_governor_observe(), mdl_governor_release(), mdl_net_get_body(), mdl_session_t::net, RA8_INTERNAL, mdl_net_resp_t::retry_after, mdl_fetch_ctx_t::session, and mdl_net_resp_t::status.

Referenced by priv_mdl_fetch_with_retry().

internal_mdl_fetch_mark_complete()

void internal_mdl_fetch_mark_complete ( mdl_fetch_ctx_t * ctx, mdl_fetch_layout_t layout, mdl_chapter_rec_t * rec, size_t * global_no )

static

Mark a chapter complete and advance combined numbering.

Performs mark complete under the injected network, governor, and storage contracts; dependency failures are propagated before incomplete bytes are published.

Parameters

[in,out]	ctx	Borrowed operation context.
[in]	layout	Selected combined or per-chapter layout.
[in,out]	rec	Persistent chapter record to mark complete.
[out]	global_no	Combined-layout page number advanced on completion.

Precondition

Required pointer arguments remain valid for the call duration.

Supplied capacities cover their referenced bounded buffers.

Postcondition

No ownership of caller-provided storage is transferred.

Result status and outputs describe one completed synchronous attempt.

Note

The function performs no dynamic allocation and retains no caller pointer.

Since

0.1.0

Definition at line 596 of file mdl_fetch.c.

References mdl_chapter_rec_t::complete, mdl_url_list_t::count, mdl_chapter_rec_t::fetched_at, mdl_fetch_ctx_t::images, k_mdl_layout_combined, mdl_chapter_rec_t::pages_done, and RA8_INTERNAL.

Referenced by internal_mdl_fetch_chapter_pages_and_checkpoint().

internal_mdl_fetch_max_u32()

uint32_t internal_mdl_fetch_max_u32 ( uint32_t a, uint32_t b )

static

Perform the max u32 step.

Performs max u32 under the injected network, governor, and storage contracts; dependency failures are propagated before incomplete bytes are published.

Parameters

[in]	a	First unsigned delay operand.
[in]	b	Second unsigned delay operand.

Returns

The bounded result computed from the supplied input.

Return values

other

The computed result in the function's declared domain.

Precondition

Required pointer arguments remain valid for the call duration.

Supplied capacities cover their referenced bounded buffers.

Postcondition

No ownership of caller-provided storage is transferred.

Result status and outputs describe one completed synchronous attempt.

Note

The function performs no dynamic allocation and retains no caller pointer.

Since

0.1.0

Definition at line 105 of file mdl_fetch.c.

References RA8_INTERNAL.

Referenced by internal_mdl_fetch_chapter_html(), and mdl_fetch_asset().

internal_mdl_fetch_page_host()

const char * internal_mdl_fetch_page_host ( const char * url, char * buf, size_t cap )

static

Governor host key for url, or NULL when it cannot be parsed.

Definition at line 207 of file mdl_fetch.c.

References mdl_url_host(), and RA8_INTERNAL.

Referenced by mdl_fetch_asset().

internal_mdl_fetch_process_chapter()

mdl_chap_status_t internal_mdl_fetch_process_chapter ( mdl_fetch_ctx_t * ctx, const char * chapter_url, mdl_fetch_layout_t layout, const char * combined_abs, const char * combined_rel, size_t * global_no, mdl_fetch_stats_t * stats, size_t chapter_index, size_t chapter_total )

static

Download one chapter (or skip it), returning its outcome.

Performs process chapter under the injected network, governor, and storage contracts; dependency failures are propagated before incomplete bytes are published.

Parameters

[in,out]	ctx	Borrowed operation context.
[in]	chapter_url	Canonical chapter URL.
[in]	layout	Selected combined or per-chapter layout.
[in]	combined_abs	Canonical combined-output directory.
[in]	combined_rel	Portable combined-output path.
[out]	global_no	Combined-layout page number advanced on completion.
[in,out]	stats	Run counters to update.
[in]	chapter_index	Zero-based chapter index.
[in]	chapter_total	Total selected chapter count.

Returns

The bounded result computed from the supplied input.

Return values

other

The computed result in the function's declared domain.

Precondition

Required pointer arguments remain valid for the call duration.

Supplied capacities cover their referenced bounded buffers.

Postcondition

No ownership of caller-provided storage is transferred.

Result status and outputs describe one completed synchronous attempt.

Note

The function performs no dynamic allocation and retains no caller pointer.

Since

0.1.0

Definition at line 844 of file mdl_fetch.c.

References internal_mdl_fetch_chapter_html(), internal_mdl_fetch_chapter_pages_and_checkpoint(), internal_mdl_fetch_select_chapter_number(), internal_mdl_fetch_select_chapter_title(), k_ch_failed, k_ch_skipped, k_ra8_ok, mdl_state_add_chapter_numbered(), mdl_urlname_chapter_parse(), mdl_urlname_last_segment(), RA8_INTERNAL, mdl_fetch_ctx_t::state, and mdl_fetch_ctx_t::update_only.

Referenced by mdl_fetch_run().

internal_mdl_fetch_resolve_dest()

bool internal_mdl_fetch_resolve_dest ( mdl_fetch_ctx_t * ctx, mdl_fetch_layout_t layout, const char * id, const char * combined_abs, const char * combined_rel, size_t global_no, char * chap_abs, size_t chap_cap, const char ** dest_abs, const char ** dest_rel, size_t * base )

static

Resolve the output directory and starting page number for one chapter.

Selects the shared combined destination or creates the chapter's contained directory, then returns the matching absolute/relative directory pointers and numbering base.

Parameters

[in,out]	ctx	Fetch context containing storage and the series root.
[in]	layout	Combined or per-chapter layout selection.
[in]	id	Sanitized chapter identifier.
[in]	combined_abs	Canonical combined-output directory.
[in]	combined_rel	Library-relative combined-output directory.
[in]	global_no	Next combined-layout page number.
[out]	chap_abs	Scratch buffer for a per-chapter absolute directory.
[in]	chap_cap	Capacity of chap_abs in bytes.
[out]	dest_abs	Receives the selected absolute directory pointer.
[out]	dest_rel	Receives the selected relative directory pointer.
[out]	base	Receives the selected starting page number.

Returns

Whether all selected paths were prepared within bounds.

Return values

true	Destination pointers and base were initialized.
false	The per-chapter directory was invalid or could not be created.

Precondition

Every pointer is non-NULL and text inputs are NUL-terminated.

chap_abs references chap_cap writable bytes.

Postcondition

Success selects only a path contained by the configured series root.

Failure publishes no destination pointer or chapter completion record.

Note

Returned path pointers remain borrowed from caller-owned storage.

Since

0.1.0

Definition at line 555 of file mdl_fetch.c.

References k_mdl_layout_combined, mdl_join_dir_under(), RA8_INTERNAL, mdl_fetch_ctx_t::series_abs_dir, and mdl_fetch_ctx_t::storage.

Referenced by internal_mdl_fetch_chapter_pages_and_checkpoint().

internal_mdl_fetch_select_chapter_number()

bool internal_mdl_fetch_select_chapter_number ( mdl_fetch_ctx_t * ctx, const char * chapter_url, mdl_chapter_work_t * work )

static

Select and strictly parse optional descriptor-driven chapter numbering.

Performs select chapter number under the injected network, governor, and storage contracts; dependency failures are propagated before incomplete bytes are published.

Parameters

[in,out]	ctx	Fetch context and retained chapter HTML.
[in]	chapter_url	Absolute chapter URL.
[in,out]	work	Chapter metadata scratch.

Returns

Whether number selection can continue.

Return values

true	The condition holds or the operation completed.
false	Input was rejected or the condition does not hold.

Precondition

Required pointer arguments remain valid for the call duration.

Supplied capacities cover their referenced bounded buffers.

Postcondition

Success preserves the recorded number or stores one complete finite match.

Malformed or over-cap matched text is logged and rejected.

Note

The function performs no dynamic allocation and retains no caller pointer.

Since

0.1.0

Definition at line 690 of file mdl_fetch.c.

References mdl_site_t::chapter_number_selector, mdl_fetch_ctx_t::diagnostic, internal_mdl_fetch_diag3(), k_mdl_title_max, k_ra8_err_invalid_size, k_ra8_err_validation_failed, k_ra8_ok, mdl_extract_selector(), mdl_urlname_chapter_text_parse(), mdl_fetch_ctx_t::page_buf, priv_mdl_fetch_record_fail(), priv_mdl_stream_text(), mdl_fetch_ctx_t::progress_error, RA8_INTERNAL, and mdl_fetch_ctx_t::site.

Referenced by internal_mdl_fetch_process_chapter().

internal_mdl_fetch_select_chapter_title()

bool internal_mdl_fetch_select_chapter_title ( mdl_fetch_ctx_t * ctx, const char * chapter_url, mdl_chapter_work_t * work )

static

Select one bounded persistent chapter title.

Performs select chapter title under the injected network, governor, and storage contracts; dependency failures are propagated before incomplete bytes are published.

Parameters

[in,out]	ctx	Fetch context and retained chapter HTML.
[in]	chapter_url	Absolute chapter URL.
[in,out]	work	Chapter metadata scratch.

Returns

Whether title selection can continue.

Return values

true	The condition holds or the operation completed.
false	Input was rejected or the condition does not hold.

Precondition

Required pointer arguments remain valid for the call duration.

Supplied capacities cover their referenced bounded buffers.

Postcondition

Success sets mdl_chapter_work_t::title.

An over-cap matched title is logged and rejected without truncation.

Note

The function performs no dynamic allocation and retains no caller pointer.

Since

0.1.0

Definition at line 625 of file mdl_fetch.c.

References mdl_site_t::chapter_title_selector, mdl_fetch_ctx_t::diagnostic, internal_mdl_fetch_diag3(), k_ra8_err_invalid_size, k_ra8_ok, mdl_extract_selector(), mdl_fetch_ctx_t::page_buf, priv_mdl_fetch_record_fail(), priv_mdl_stream_text(), mdl_fetch_ctx_t::progress_error, RA8_INTERNAL, and mdl_fetch_ctx_t::site.

Referenced by internal_mdl_fetch_process_chapter().

internal_mdl_fetch_tally()

bool internal_mdl_fetch_tally ( mdl_chap_status_t s, mdl_fetch_stats_t * stats )

static

Fold one chapter outcome into run statistics.

Increments exactly one completed, skipped, or failed counter and reports whether the outcome should fail the enclosing run.

Parameters

[in]	s	Completed, skipped, or failed chapter outcome.
[in,out]	stats	Run counters to update.

Returns

Whether s represents failure.

Return values

true	The failed counter was incremented.
false	The completed or skipped counter was incremented.

Precondition

stats is non-NULL.

The selected counter can be incremented without size_t overflow.

Postcondition

Exactly one chapter counter is incremented once.

Page and byte counters remain unchanged.

Note

Unknown enum values are conservatively counted as failures.

Since

0.1.0

Definition at line 908 of file mdl_fetch.c.

References mdl_fetch_stats_t::chapters_completed, mdl_fetch_stats_t::chapters_failed, mdl_fetch_stats_t::chapters_skipped, k_ch_completed, k_ch_failed, k_ch_skipped, and RA8_INTERNAL.

Referenced by mdl_fetch_run().

mdl_fetch_asset()

ra8_err_t mdl_fetch_asset	(	mdl_fetch_ctx_t *	ctx,
		const char *	url,
		const char *	target_abs,
		const char *	referer,
		mdl_net_resp_t *	out_resp,
		size_t *	out_bytes )

Fetch one policy-governed asset to an absolute file path.

Provides non-page callers (notably series-cover acquisition) the same robots gate, backend SSRF policy, per-host governor, bounded retry loop, and atomic publication used by chapter images. The transfer is staged beside the target; an existing target survives every failed or zero-byte response unchanged.

Parameters

[in,out]	ctx	Injected session/site/governor dependencies.
[in]	url	Absolute HTTP(S) asset URL (never NULL).
[in]	target_abs	Absolute destination path (never NULL).
[in]	referer	Referer header, or NULL to omit it.
[out]	out_resp	Finished response metadata, or NULL.
[out]	out_bytes	Committed byte count, or NULL.

Returns

An ra8_err_t transfer result.

Return values

k_ra8_ok	A non-empty asset was atomically committed.
k_ra8_err_invalid_arg	Required context/path/URL input was invalid.
k_ra8_err_invalid_size	The server returned a successful empty body.
k_ra8_fail	Robots refused the URL or publication failed.

Precondition

ctx has a session, site descriptor, and configured network backend.

target_abs names a writable absolute path whose parent exists.

Postcondition

On success, target_abs holds exactly *out_bytes nonzero bytes.

On failure, a pre-existing target_abs is unchanged and no temp remains.

Note

Not thread-safe: mutates the session cache and governor.

Since

0.1.0

Definition at line 356 of file mdl_fetch.c.

References mdl_site_t::img_delay_max, mdl_site_t::img_delay_min, internal_fetch_asset_execute(), internal_mdl_fetch_max_u32(), internal_mdl_fetch_page_host(), k_mdl_gov_host_max, k_ra8_err_invalid_arg, k_ra8_fail, mdl_session_url_allowed(), mdl_session_t::net, priv_mdl_fetch_record_fail(), mdl_fetch_ctx_t::session, mdl_fetch_ctx_t::site, mdl_fetch_ctx_t::timeout_ms, and mdl_session_t::user_agent.

mdl_fetch_run()

ra8_err_t mdl_fetch_run	(	mdl_fetch_ctx_t *	ctx,
		const mdl_url_list_t *	chapters,
		mdl_fetch_layout_t	layout,
		const char *	combined_dir_rel,
		mdl_fetch_stats_t *	stats )

Download a series' chapters incrementally, resuming and deduping.

Walks chapters in order. For each chapter it derives a stable identifier, finds-or-adds its state record, and – unless update_only is set and the record is already complete – fetches the chapter page, extracts its image URLs, and writes each page under ctx->series_abs_dir. A page already held (matched by source-URL hash and verified by content hash) is reused from disk rather than re-fetched, including across chapters. Each request is retried up to a small bounded number of times on a retryable outcome (a transport error, timeout, 429 or 5xx), always paced through the politeness governor so a retry never becomes an unpaced hammer; a 404 is never retried. A chapter is marked complete and its record checkpointed only once every page is present and verified; a page failure leaves the chapter partial for the next run to resume, and is appended to ctx->faillog (when set) with its URL and status. Per-page progress is emitted through ctx->progress_fn (when set). In k_mdl_layout_combined the pages of all chapters share one directory with numbering continued across chapters (derived from recorded per-chapter page counts, so a resume reproduces it); k_mdl_layout_separate gives each chapter its own directory numbered from one.

Parameters

[in,out]	ctx	Injected dependencies and scratch (never NULL).
[in]	chapters	Live, ordered chapter URL list (never NULL).
[in]	layout	Output directory layout.
[in]	combined_dir_rel	Directory name (relative to the series dir) for k_mdl_layout_combined; ignored otherwise.
[out]	stats	Receives the run tallies (never NULL).

Returns

An ra8_err_t summarising the run.

Return values

k_ra8_ok	Every attempted chapter completed.
k_ra8_err_invalid_arg	A required pointer argument was NULL.
k_ra8_fail	At least one chapter was left partial/failed.

Precondition

ctx, chapters, stats are non-NULL and ctx is fully populated.

ctx->series_abs_dir exists and is an absolute resolved path.

Postcondition

stats reflects the run; pages_fetched + pages_reused accounts for every page of every completed chapter.

ctx->state is updated and, when ctx->state_path is set, checkpointed atomically after each page and each completed chapter.

Note

Not thread-safe: mutates shared state, scratch and the filesystem.

See also

mdl_state_find_page

Since

0.1.0

Definition at line 947 of file mdl_fetch.c.

References mdl_url_list_t::count, internal_mdl_fetch_ctx_ready(), internal_mdl_fetch_process_chapter(), internal_mdl_fetch_tally(), k_mdl_layout_combined, k_ra8_err_invalid_arg, k_ra8_fail, k_ra8_ok, mdl_join_dir_under(), memset(), mdl_fetch_ctx_t::progress_error, mdl_fetch_ctx_t::series_abs_dir, mdl_fetch_ctx_t::storage, and mdl_url_list_t::urls.

Referenced by internal_run_prepared().

priv_mdl_fetch_cache_get_buf()

ra8_err_t priv_mdl_fetch_cache_get_buf	(	void *	context,
		const char *	url,
		const mdl_net_req_t *	request,
		char *	buffer,
		size_t	capacity,
		size_t *	out_length,
		mdl_net_resp_t *	response )

Fetch one cache attempt through governor and bounded retry policy.

Fetch one cache buffer through governor and bounded retry policy.

Parameters

[in,out]	context	mdl_fetch_cache_request_t state.
[in]	url	Exact chapter URL.
[in]	request	Conditional request metadata.
[out]	buffer	Bounded HTML destination.
[in]	capacity	Writable destination capacity.
[out]	out_length	Exact received bytes.
[out]	response	Finished response metadata.

Returns

Canonical governed network status.

Precondition

Every pointer is non-NULL and the context is fully initialized.

buffer spans capacity writable bytes.

Postcondition

Every attempt is acquired, observed, and released exactly once.

Retryable results make at most k_fetch_max_attempts attempts.

Note

No callback argument is retained.

Since

0.1.0

Definition at line 421 of file mdl_fetch.c.

References mdl_fetch_cache_request::ctx, mdl_fetch_ctx_t::gov, mdl_fetch_cache_request::host, mdl_fetch_cache_request::jmax, mdl_fetch_cache_request::jmin, k_fetch_max_attempts, k_ra8_fail, k_ra8_ok, mdl_governor_acquire(), mdl_governor_observe(), mdl_governor_release(), mdl_net_get_buf(), mdl_session_t::net, priv_mdl_fetch_is_retryable(), RA8_PRIV, mdl_net_resp_t::retry_after, mdl_fetch_ctx_t::session, and mdl_net_resp_t::status.

Referenced by internal_cover_cached(), and internal_mdl_fetch_chapter_html().

priv_mdl_fetch_checkpoint()

ra8_err_t priv_mdl_fetch_checkpoint

(

const mdl_fetch_ctx_t *

ctx

)

Persist state atomically when a checkpoint target is configured.

Persist the current fetch state when a checkpoint path is configured.

Definition at line 193 of file mdl_fetch.c.

References k_ra8_ok, mdl_state_save(), priv_mdl_fetch_record_fail(), RA8_PRIV, mdl_fetch_ctx_t::state, mdl_fetch_ctx_t::state_path, and mdl_fetch_ctx_t::storage.

Referenced by internal_mdl_fetch_chapter_pages_and_checkpoint(), and internal_mdl_fetch_one_page().

priv_mdl_fetch_is_retryable()

bool priv_mdl_fetch_is_retryable

(

ra8_err_t

rc

)

Whether a failed transfer result is worth retrying.

The retry classifier the bounded per-request loop consults. A transport error (k_ra8_fail), a timeout (k_ra8_err_timeout) and a throttle (k_ra8_err_busy, i.e. HTTP 429/503) are transient and retried; every other outcome – success, an absent resource (k_ra8_err_not_found / 404), an over-cap body (k_ra8_err_no_mem), a refused argument or a governor decline – is terminal and breaks the loop. Retryability is decided on the real classified status (priv_mdl_net_curl_classify), never on a collapsed generic failure, so a 404 is never retried while a 503 is.

Parameters

[in]

rc

The transfer's classified ra8_err_t result.

Returns

Whether the request should be retried.

Return values

true	rc is k_ra8_err_busy, k_ra8_err_timeout or k_ra8_fail.
false	Any other value (success or a permanent error).

Precondition

rc is a value from the ra8_err_t contract.

The caller bounds the retry count independently (this makes no loop).

Postcondition

No state is modified.

k_ra8_ok always yields false, so a success ends the loop.

Note

Thread-safe: depends only on its argument.

MC/DC:

Decision: (rc == k_ra8_err_busy) || (rc == k_ra8_err_timeout) || (rc == k_ra8_fail) (3 conditions). Cited as apps/shared_libs/mdl/src/mdl_fetch.c@priv_mdl_fetch_is_retryable.

• Vector 1: rc=k_ra8_err_not_found -> false (control: all three false)
• Vector 2: rc=k_ra8_err_busy -> true (varies condition 1)
• Vector 3: rc=k_ra8_err_timeout -> true (varies condition 2)
• Vector 4: rc=k_ra8_fail -> true (varies condition 3) Vectors 1+2 prove condition 1's independent influence, 1+3 condition 2's, 1+4 condition 3's. N+1 = 4 vectors for N=3 conditions: minimal MC/DC.

Since

0.1.0

Definition at line 110 of file mdl_fetch.c.

References k_ra8_err_busy, k_ra8_err_timeout, k_ra8_fail, and RA8_PRIV.

Referenced by internal_discover_fetch(), priv_mdl_fetch_cache_get_buf(), and priv_mdl_fetch_with_retry().

priv_mdl_fetch_reason()

void priv_mdl_fetch_reason	(	ra8_err_t	err,
		long	status,
		char *	buf,
		size_t	cap )

Render a human-readable reason for a transfer result and HTTP status.

Maps the classified ra8_err_t (plus the observed HTTP status, where one applies) to a short prose reason, so a failure is reported as "rate limited (HTTP 503)" rather than a bare err 0x109. The HTTP status is appended in parentheses when it is non-zero. Used by the CLI for both the per-failure summary and any inline error line.

Parameters

[in]	err	The classified transfer result.
[in]	status	HTTP status observed, or 0 when none applies.
[out]	buf	Destination buffer for the NUL-terminated reason.
[in]	cap	Capacity of buf in bytes.

Returns

Nothing.

Precondition

buf is non-NULL and cap > 0 for output to be written.

err comes from a finished transfer.

Postcondition

buf is NUL-terminated when cap > 0.

A NULL buf or zero cap is a tolerated no-op.

Note

Thread-safe: writes only the caller-provided buffer.

Since

0.1.0

Definition at line 130 of file mdl_fetch.c.

References internal_mdl_fetch_fail_reason(), k_ra8_err_busy, k_ra8_err_no_mem, k_ra8_err_not_found, k_ra8_err_retry_limit, k_ra8_err_timeout, k_ra8_fail, k_ra8_ok, and RA8_PRIV.

Referenced by internal_download_page_image(), internal_extract_page_images(), internal_finish_artifact_fetch(), mdl_discover_run(), and mdl_report_failures().

priv_mdl_fetch_record_fail()

void priv_mdl_fetch_record_fail	(	const mdl_fetch_ctx_t *	ctx,
		const char *	url,
		long	status,
		ra8_err_t	err )

Append one failure to the run's log (when set); always tally total.

Record one classified failure in the caller's bounded log.

Definition at line 175 of file mdl_fetch.c.

References mdl_fetch_fail_t::err, mdl_fetch_ctx_t::faillog, k_mdl_fetch_fail_max, mdl_fetch_fail_t::status, and mdl_fetch_fail_t::url.

Referenced by internal_cover_cached(), internal_cover_stream(), internal_fetch_asset_execute(), internal_mdl_fetch_chapter_html(), internal_mdl_fetch_prepare_page(), internal_mdl_fetch_publish_page(), internal_mdl_fetch_resolve_not_modified(), internal_mdl_fetch_select_chapter_number(), internal_mdl_fetch_select_chapter_title(), mdl_fetch_asset(), and priv_mdl_fetch_checkpoint().

priv_mdl_fetch_run_incomplete()

bool priv_mdl_fetch_run_incomplete

(

const mdl_fetch_stats_t *

stats

)

Whether a finished run left anything unfetched (chapter or page).

The single honesty predicate the CLI's export gate reads: a run is incomplete when any chapter was left partial OR any individual page failed. It exists so the "do not package an incomplete archive" decision is one tested function rather than a compound test re-derived at each export site.

Parameters

[in]

stats

The run tallies (may be NULL).

Returns

Whether the run has an unfetched chapter or page.

Return values

true	chapters_failed > 0 or pages_failed > 0.
false	A NULL stats, or a run in which nothing failed.

Precondition

stats, when non-NULL, was filled by mdl_fetch_run.

The caller treats NULL as "cannot prove incomplete" (false).

Postcondition

No state is modified.

A fully clean run yields false.

Note

Thread-safe: reads only its argument.

MC/DC:

Decision: (stats->chapters_failed > 0) || (stats->pages_failed > 0) (2 conditions). Cited as apps/shared_libs/mdl/src/mdl_fetch.c@priv_mdl_fetch_run_incomplete.

• Vector 1: chapters_failed=0, pages_failed=0 -> false (both false)
• Vector 2: chapters_failed=1, pages_failed=0 -> true (varies chapters)
• Vector 3: chapters_failed=0, pages_failed=1 -> true (varies pages) Vectors 1+2 prove the chapter condition's influence, 1+3 the page condition's. N+1 = 3 vectors for N=2 conditions: minimal MC/DC.

Since

0.1.0

Definition at line 115 of file mdl_fetch.c.

References mdl_fetch_stats_t::chapters_failed, mdl_fetch_stats_t::pages_failed, and RA8_PRIV.

Referenced by mdl_pack_combined_meta().

priv_mdl_fetch_with_retry()

ra8_err_t priv_mdl_fetch_with_retry	(	mdl_fetch_ctx_t *	ctx,
		const char *	host,
		const char *	url,
		const mdl_net_req_t *	req,
		mdl_net_body_sink_t *	sink,
		uint32_t	jmin,
		uint32_t	jmax,
		mdl_net_resp_t *	out_resp,
		size_t *	out_bytes )

Bounded, governed retry of one image transfer; last status latched.

Perform one bounded governed file transfer with retry classification.

Definition at line 264 of file mdl_fetch.c.

References internal_mdl_fetch_governed_get_body(), k_fetch_max_attempts, k_ra8_fail, priv_mdl_fetch_is_retryable(), and RA8_PRIV.

Referenced by internal_cover_stream(), internal_fetch_asset_execute(), internal_mdl_fetch_prepare_page(), and internal_mdl_fetch_resolve_not_modified().