mdl_robots.h

Go to the documentation of this file.

#pragma once
 
#include <stddef.h>
#include <stdint.h>

typedef enum : uint16_t {
  k_mdl_robots_max_rules = 64,  
  k_mdl_robots_path_max  = 256, 
} mdl_robots_limits_t;

typedef enum : uint16_t {
  k_mdl_robots_max_hosts = 16,  
  k_mdl_robots_host_max  = 128, 
} mdl_robots_cache_limits_t;

typedef enum : uint8_t {
  k_mdl_rule_disallow = 0, 
  k_mdl_rule_allow    = 1, 
} mdl_robots_rule_kind_t;

typedef enum : uint8_t {
  k_mdl_robots_fetch_ok     = 0, 
  k_mdl_robots_fetch_absent = 1, 
  k_mdl_robots_fetch_denied = 2, 
} mdl_robots_fetch_result_t;

typedef struct {
  mdl_robots_rule_kind_t kind;                        
  uint16_t               len;                         
  char                   path[k_mdl_robots_path_max]; 
} mdl_robots_rule_t;

typedef struct {
  mdl_robots_rule_t rules[k_mdl_robots_max_rules]; 
  size_t            count;                         
  bool              valid;                         
  bool              have_crawl_delay;              
  uint32_t          crawl_delay_ms;                
} mdl_robots_t;

void mdl_robots_parse(const char* text, size_t len, const char* ua_token, mdl_robots_t* out);

bool mdl_robots_allows(const mdl_robots_t* robots, const char* path);

const char* mdl_robots_disallow_reason(const mdl_robots_t* robots, const char* path);

typedef struct {
  char         scheme[6];                   
  char         host[k_mdl_robots_host_max]; 
  bool         used;                        
  bool         disallow_all;                
  mdl_robots_t rules;                       
} mdl_robots_cache_entry_t;

typedef struct {
  mdl_robots_cache_entry_t hosts[k_mdl_robots_max_hosts]; 
  mdl_robots_cache_entry_t overflow;                      
} mdl_robots_cache_t;

typedef mdl_robots_fetch_result_t (
  *mdl_robots_fetch_fn)(void* ctx, const char* robots_url, char* buf, size_t cap, size_t* out_len);

const mdl_robots_t* mdl_robots_cache_consult(mdl_robots_cache_t* cache,
                                             const char*         scheme,
                                             const char*         host,
                                             const char*         ua_token,
                                             mdl_robots_fetch_fn fetch,
                                             void*               ctx,
                                             char*               scratch,
                                             size_t              scratch_cap);