mdl_fetch.h

Go to the documentation of this file.

#pragma once
 
#include <stddef.h>
#include <stdint.h>
 
#include "mdl_cache.h"
#include "mdl_config.h"
#include "mdl_extract.h"
#include "mdl_politeness.h"
#include "mdl_session.h"
#include "mdl_state.h"
#include "mdl_storage.h"
#include "ra8_err.h"
#include "ra8_io_stream.h"

typedef enum : uint8_t {
  k_mdl_layout_combined = 0, 
  k_mdl_layout_separate = 1, 
} mdl_fetch_layout_t;

typedef struct {
  size_t chapters_completed; 
  size_t chapters_skipped;   
  size_t chapters_failed;    
  size_t pages_fetched;      
  size_t pages_reused;       
  size_t pages_failed;       
} mdl_fetch_stats_t;

typedef enum : uint16_t {
  k_mdl_fetch_fail_max = 256, 
} mdl_fetch_faillog_size_t;

typedef struct {
  char      url[k_mdl_url_max]; 
  long      status;             
  ra8_err_t err;                
} mdl_fetch_fail_t;

typedef struct {
  mdl_fetch_fail_t items[k_mdl_fetch_fail_max]; 
  size_t           count;                       
  size_t           total;                       
} mdl_fetch_faillog_t;

typedef struct {
  size_t      chapter_index; 
  size_t      chapter_total; 
  const char* chapter_id;    
  size_t      page_index;    
  size_t      page_total;    
  uint64_t    page_bytes;    
  uint32_t    elapsed_ms;    
  bool        reused;        
} mdl_fetch_progress_t;

typedef ra8_err_t (*mdl_progress_fn)(void* ctx, const mdl_fetch_progress_t* ev);

typedef struct {
  mdl_session_t*       session;        
  mdl_storage_t*       storage;        
  mdl_state_t*         state;          
  mdl_cache_t*         cache;          
  const char*          state_path;     
  const char*          series_abs_dir; 
  const char*          series_url;     
  const mdl_site_t*    site;           
  mdl_governor_t*      gov;            
  uint32_t             timeout_ms;     
  char*                page_buf;       
  size_t               page_cap;       
  mdl_url_list_t*      images;         
  bool                 update_only;    
  bool                 refetch;        
  mdl_fetch_faillog_t* faillog;        
  ra8_io_stream_t*     diagnostic;     
  mdl_progress_fn      progress_fn;    
  void*                progress_ctx;   
  ra8_err_t            progress_error; 
} mdl_fetch_ctx_t;

ra8_err_t mdl_fetch_asset(mdl_fetch_ctx_t* ctx,
                          const char*      url,
                          const char*      target_abs,
                          const char*      referer,
                          mdl_net_resp_t*  out_resp,
                          size_t*          out_bytes);

ra8_err_t mdl_fetch_run(mdl_fetch_ctx_t*      ctx,
                        const mdl_url_list_t* chapters,
                        mdl_fetch_layout_t    layout,
                        const char*           combined_dir_rel,
                        mdl_fetch_stats_t*    stats);