fetch online data

CMakeLists.txt

@@ -40,12 +40,17 @@ else (ROOT_FOUND)
   MESSAGE(">> ROOT not found")
 endif (ROOT_FOUND)
 
+find_package(CURL REQUIRED)
+if (CURL_FOUND)
+  add_definitions(-DWITH_CURL)
+endif (CURL_FOUND)
 
 include(GNUInstallDirs OPTIONAL)
-set(CMAKE_INSTALL_LIBDIR lib CACHE PATH "install libdir")
-set(CMAKE_INSTALL_INCLUDEDIR include CACHE PATH "install includedir")
+# set(CMAKE_INSTALL_LIBDIR lib CACHE PATH "install libdir")
+# set(CMAKE_INSTALL_INCLUDEDIR include CACHE PATH "install includedir")
 
 add_subdirectory(src)
+add_subdirectory(apps)
 
 ### enable ctest
 include(CTest)
@@ -53,3 +58,9 @@ enable_testing()
 add_subdirectory(test)
 
 message(">> CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
+
+# Add uninstall target
+# Requirements: Copy the uninstall.cmake file to the appropriate CMAKE_MODULE_PATH.
+add_custom_target(uninstall
+    "${CMAKE_COMMAND}" -P "${CMAKE_SOURCE_DIR}/cmake/uninstall.cmake"
+)

apps/CMakeLists.txt

+add_executable(crdb-bin crdb.cc)
+target_link_libraries(crdb-bin PUBLIC crdb PUBLIC ${ROOT_LIBRARIES})
+set_target_properties(crdb-bin PROPERTIES OUTPUT_NAME crdb)
+
+install(TARGETS crdb-bin
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})

apps/argument

+/* 
+ * yuc/argument: a useable command line argument parser
+ * 
+ * #include <yuc/argument>
+ * int main(int argc, char* argv[]) {
+ *   yuc:argument arg;
+ *   arg.add_flag('v', "verbose", "print more infomation");
+ *   arg.add_option('i', "input", "inputfile", "/dev/stdin");
+ *   arg.add_option('n', "size",  "specific size", "-1");
+ *   if(arg.parse(argc, argv)) { return 1; }
+ *   std::ifstream ifs(arg["input"]);
+ *   size_t n = arg["size"];
+ *   bool verbose = arg["input"];
+ *   //...
+ * }
+ *
+ */
+
+#pragma once
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+namespace yuc {
+class argument {
+
+  protected:
+    class option {
+      public:
+        // Cannot have const member for 'std::sort' to work
+        char short_name;
+        std::string long_name;
+        std::string hint;
+        std::string value;
+        bool boolean; // TODO: adapt entire code with this flag
+        bool valid;
+
+      public:
+        template <typename T, typename = typename std::enable_if<
+                                  !(std::is_same<T, bool>::value) &&
+                                  !(std::is_same<T, char>::value) &&
+                                      std::is_arithmetic<T>::value,
+                                  T>::type>
+        operator T(void) const {
+            T v;
+            std::istringstream(value) >> v;
+            // TODO: validation
+            return v;
+        }
+        operator std::string&(void) { return value; }
+        operator const std::string&(void) const { return value; }
+        operator bool(void) const { return !value.empty() && value != "0"; }
+        const std::string& str(void) const { return value; }
+
+        friend std::ostream& operator<<(std::ostream& os, const option& opt) {
+            return os << opt.value;
+        }
+
+        struct less {
+            bool operator()(const option& o1, const option& o2) const {
+                char s1 = o1.short_name % 32, s2 = o2.short_name % 32;
+                if (s1 && s2) {
+                    return s1 == s2 ? o1.short_name < o2.short_name
+                                    : s1 < o2.short_name % 32;
+                } else if (s1 || s2) {
+                    return (s1) ? (s1 <= o2.long_name.front() % 32)
+                                : (o1.long_name.front() % 32 < s1);
+                } else {
+                    std::string l1 = o1.long_name, l2 = o2.long_name;
+                    std::transform(l1.begin(), l1.end(), l1.begin(), ::toupper);
+                    std::transform(l2.begin(), l2.end(), l2.begin(), ::toupper);
+                    return l1 == l2 ? o1.long_name < o2.long_name : l1 < l2;
+                }
+            }
+        };
+    }; // class option
+
+  protected:
+    std::vector<option> _options;
+    std::vector<option> _flags;
+    std::vector<std::string> _position;
+    std::vector<std::string> _usages;
+    std::string _help_str;
+
+    bool _parsed = false;
+
+  protected:
+    // tried to use std::optional<option&> at first without success
+    // switched to pointer after reading:
+    //   https://foonathan.net/blog/2018/07/12/optional-reference.html
+    option* find_option(char short_name) {
+        for (auto& o : _options)
+            if (o.short_name == short_name)
+                return &o;
+        return nullptr;
+    }
+    const option* find_option(char short_name) const {
+        for (const auto& o : _options)
+            if (o.short_name == short_name)
+                return &o;
+        return nullptr;
+    }
+    option* find_flag(char short_name) {
+        for (auto& o : _flags)
+            if (o.short_name == short_name)
+                return &o;
+        return nullptr;
+    }
+    const option* find_flag(char short_name) const {
+        for (const auto& o : _flags)
+            if (o.short_name == short_name)
+                return &o;
+        return nullptr;
+    }
+    option* find_option(const std::string& long_name) {
+        for (auto& o : _options)
+            if (o.long_name == long_name)
+                return &o;
+        return nullptr;
+    }
+    const option* find_option(const std::string& long_name) const {
+        for (const auto& o : _options)
+            if (o.long_name == long_name)
+                return &o;
+        return nullptr;
+    }
+    option* find_flag(const std::string& long_name) {
+        for (auto& o : _flags)
+            if (o.long_name == long_name)
+                return &o;
+        return nullptr;
+    }
+    const option* find_flag(const std::string& long_name) const {
+        for (const auto& o : _flags)
+            if (o.long_name == long_name)
+                return &o;
+        return nullptr;
+    }
+
+  public: // Helper section
+    void add_usage(const std::string& usage_str) {
+        _usages.push_back(usage_str);
+    }
+    void add_help(const std::string& help_str) { _help_str = help_str; }
+    void help(std::ostream& os = std::cout) {
+        for (const auto& u : _usages) {
+            os << "Usage:  " << _position[0];
+            if (!u.empty()) {
+                os << " " << u;
+            }
+            if (u.back() != '\n') {
+                // What if on Windows?
+                os << std::endl;
+            }
+        }
+        if (!_help_str.empty()) {
+            os << _help_str;
+            if (_help_str.back() != '\n') {
+                os << std::endl;
+            }
+        } else {
+            std::stringstream ss;
+            ss << std::left;
+            std::vector<option> all_options = _options;
+            all_options.insert(all_options.end(), _flags.cbegin(),
+                               _flags.cend());
+            std::sort(all_options.begin(), all_options.end(), option::less());
+            ss << "Options:" << std::endl;
+            size_t maxl = 0;
+            for (const auto& opt : all_options) {
+                maxl = std::max(maxl,
+                                opt.long_name.length() +
+                                    (opt.boolean ? 0 : 1 + opt.value.length()));
+            }
+            maxl += 1;
+            for (const auto& opt : all_options) {
+                if (opt.short_name % 32) {
+                    ss << "  -" << opt.short_name;
+                    if (opt.long_name.empty()) {
+                        ss << ' ' << std::setw(maxl + 3);
+                    } else {
+                        ss << ", --" << std::setw(maxl);
+                    }
+                } else {
+                    ss << std::right << std::setw(8) << "--" << std::left
+                       << std::setw(maxl);
+                }
+                ss << opt.long_name +
+                          (opt.boolean ? ""
+                                       : (opt.long_name.empty() ? ' ' : '=') +
+                                             opt.value)
+                   << opt.hint << std::endl;
+            }
+            os << ss.str();
+        }
+    }
+
+  public:
+    // TODO: duplication check
+    void add_option(char short_name, const std::string& long_name,
+                    const std::string& hint) {
+        _options.push_back({short_name, long_name, hint, "", false, false});
+    }
+    void add_option(char short_name, const std::string& long_name,
+                    const std::string& hint, const std::string& default_value) {
+        _options.push_back(
+            {short_name, long_name, hint, default_value, false, true});
+    }
+    template <typename T>
+    void add_option(char short_name, const std::string& long_name,
+                    const std::string& hint, const T& default_value) {
+        std::stringstream ss;
+        ss << std::setprecision(16) << default_value;
+        _options.push_back(
+            {short_name, long_name, hint, ss.str(), false, true});
+    }
+    void add_flag(char short_name, const std::string& long_name,
+                  const std::string& hint, bool default_value = false) {
+        _flags.push_back({short_name, long_name, hint,
+                          default_value ? "1" : "0", true, true});
+    }
+
+    size_t size(void) const { return _position.size(); }
+    const std::string& operator[](size_t i) const {
+        if (i >= _position.size())
+            throw std::logic_error("positional index out of range");
+        else
+            return _position[i];
+    }
+    std::string& operator[](size_t i) {
+        if (i >= _position.size())
+            throw std::logic_error("positional index out of range");
+        else
+            return _position[i];
+    }
+
+    const option& operator[](const std::string& name) const {
+        auto opt = find_option(name);
+        if (!opt) {
+            opt = find_flag(name);
+        }
+        if (opt) {
+            return *opt;
+        } else {
+            std::stringstream ss;
+            ss << "unknown option '" << name << "'";
+            throw std::logic_error(ss.str());
+        }
+    }
+    option& operator[](const std::string& name) {
+        auto opt = find_option(name);
+        if (!opt) {
+            opt = find_flag(name);
+        }
+        if (opt) {
+            return *opt;
+        } else {
+            std::stringstream ss;
+            ss << "unknown option '" << name << "'";
+            throw std::logic_error(ss.str());
+        }
+    }
+
+  public: // open validation
+    template <size_t n> struct at_least {
+        bool operator()(const argument& arg) const {
+            return arg._position.size() > n;
+        }
+        const char* what() const { return "missing operand"; }
+    };
+
+    template <size_t n> struct at_most {
+        bool operator()(const argument& arg) const {
+            return arg._position.size() <= n + 1;
+        }
+        const char* what() const { return "extra operand"; }
+    };
+
+    template <size_t n> struct equal_to {
+        bool operator()(const argument& arg) const {
+            return arg._position.size() == n + 1;
+        }
+        const char* what() const { return "unmatched number of operands"; }
+    };
+
+  public:
+    template <typename UnaryOperation = argument::at_least<0>>
+    bool parse(int argc, char** argv,
+               UnaryOperation valid_op = UnaryOperation()) {
+        if (_parsed) {
+            throw std::logic_error("do not parse again");
+        }
+        _parsed = true;
+        add_flag('h', "help", "print this helpful message", false);
+        _position.clear();
+        _position.emplace_back(argv[0]);
+
+        bool invalid = true;
+        for (size_t i = 1; i < argc; ++i) {
+            option* o;
+
+            const std::string a = argv[i];
+            if (a.substr(0, 2) == "--") {
+                if (a.length() == 2) { // --
+                    while (++i < argc) {
+                        _position.emplace_back(argv[i]);
+                    }
+                    break;
+                } else {
+                    if (a.substr(2, 3) == "no-" &&
+                        (o = find_flag(a.substr(5)))) {
+                        // clear flag
+                        o->value = "0";
+                        o->valid = true;
+                        continue;
+                    } else if (o = find_flag(a.substr(2))) {
+                        // set flag
+                        o->value = "1";
+                        o->valid = true;
+                        continue;
+                    } else {
+                        size_t eqpos = a.find_first_of('=');
+                        if (eqpos == a.npos && (o = find_option(a.substr(2)))) {
+                            // seperated long argument
+                            if (++i == argc) {
+                                std::cerr
+                                    << _position[0]
+                                    << ": option requires an argument -- '" +
+                                           a + '\''
+                                    << std::endl;
+                                goto invalid_arg;
+                            } else {
+                                o->value = argv[i];
+                                o->valid = true;
+                                continue;
+                            }
+                        } else if (o = find_option(a.substr(2, eqpos - 2))) {
+                            // joined long argument
+                            o->value = a.substr(eqpos + 1);
+                            o->valid = true;
+                            continue;
+                        } else {
+                            std::cerr << _position[0]
+                                      << ": unrecognized option '" << a << "'"
+                                      << std::endl;
+                            goto invalid_arg;
+                        }
+                    }
+                }
+            } else if (a.length() > 1 && a[0] == '-') {
+                // short arguments
+                for (size_t j = 1; j < a.length(); ++j) {
+                    if (o = find_flag(a[j])) {
+                        o->value = "1";
+                        o->valid = true;
+                        continue;
+                    } else if (o = find_option(a[j])) {
+                        if (++j == a.length()) {
+                            // end of string, read next argument
+                            if (++i == argc) {
+                                std::cerr
+                                    << _position[0]
+                                    << ": option requires an argument -- '"
+                                    << a[--j] << '\'' << std::endl;
+                                goto invalid_arg;
+                            } else {
+                                o->value = argv[i];
+                                o->valid = true;
+                            }
+                        } else {
+                            // read the rest
+                            o->value = a.substr(j);
+                            o->valid = true;
+                        }
+                        break; // out of current argv[i]
+                    } else {
+                        std::cerr << _position[0] << ": invalid option -- '"
+                                  << a[j] << "'" << std::endl;
+                        goto invalid_arg;
+                    }
+                }
+            } else {
+                _position.emplace_back(argv[i]);
+                continue;
+            }
+        }
+
+        invalid = false;
+    invalid_arg:
+        if (invalid || *find_flag('h')) {
+            invalid = true;
+        } else if (!valid_op(*this)) {
+            std::cerr << _position[0] << ": " << valid_op.what() << std::endl;
+            invalid = true;
+        } else {
+            for (const auto& opt : _options) {
+                if (!opt.valid) {
+                    invalid = true;
+                    std::string name;
+                    if (opt.short_name % 32 == 0) {
+                        name = "--" + opt.long_name;
+                    } else if (opt.long_name.empty()) {
+                        name = '-' + opt.short_name;
+                    } else {
+                        name = '-' + opt.short_name + "/--" + opt.long_name;
+                    }
+                    std::cerr << _position[0] << ": require option '" << name
+                              << "'" << std::endl;
+                }
+            }
+        }
+
+        if (invalid) {
+            this->help();
+        }
+        return !invalid;
+    }
+};
+
+}; // namespace yuc
+
+#define _YUC_ARGUMENT_PARSE_ASSERTION(arg, argc, argv) \
+    {                                                  \
+        if (!arg.parse(argc, argv)) {                  \
+            return EXIT_FAILURE;                       \
+        }                                              \
+    }
+
+// vi:ft=cpp

apps/crdb.cc

+#include "argument"
+#include <crdb.h>
+#include <cstdlib> // for getenv
+#include <fstream>
+#include <iostream>
+
+int main(int argc, char* argv[]) {
+
+    std::string cache_dir;
+    if (const char* env_p = std::getenv("CRDB_CACHE_DIR")) {
+        cache_dir = env_p;
+    } else {
+        cache_dir = "/tmp/crdb_cache";
+    }
+
+    yuc::argument arg;
+
+    arg.add_flag('g', "get", "get data from CRDB", false);
+    arg.add_option('t', "etype", "energy type [EKN],EK,R,ETOT,ETOTN", "EKN");
+    arg.add_option('s', "subexp",
+                   "name (and time intervals) of sub-experiments", "");
+    arg.add_option('o', "output", "output filename", "");
+    arg.add_flag('a', "append", "append to output file", false);
+
+    arg.add_usage("[crplot script...]");
+    arg.add_usage(
+        "-g [-t ETYPE] [-s SUBEXP] [-o FILE] <numerator> [denominator]");
+
+    _YUC_ARGUMENT_PARSE_ASSERTION(arg, argc, argv);
+
+    if (arg["get"]) {
+        CRRequest req;
+
+        switch (arg.size()) {
+        case 2: {
+            req.num = arg[1];
+            break;
+        }
+        case 3: {
+            req.num = arg[1];
+            req.den = arg[2];
+            break;
+        }
+        default: {
+            std::cerr << arg[0] << ": CR quantity not provided" << std::endl;
+            arg.help();
+            return EXIT_FAILURE;
+        }
+        }
+
+        req.energy_type    = arg["etype"].str();
+        std::string subexp = arg["subexp"];
+        std::string ofn    = arg["output"];
+        if (ofn == "-") {
+            ofn = "/dev/stdout";
+        }
+
+        if (ofn.empty()) {
+            // list experiments
+            auto dataset = req.fetch(cache_dir);
+            for (auto pdata : dataset) {
+                auto& d = *pdata;
+                if (subexp.empty() ||
+                    d.experiment.substr(0, subexp.size()) == subexp)
+                    // y, x, yrange, size, exp, cite
+                    std::printf(
+                        "%s   %s    %.1e ~ %.1e   %2ld    %-40s  %s\n", //
+                        d.y_quantity.c_str(), d.x_quantity.c_str(), d.x().min(),
+                        d.x().max(), d.size(), d.experiment.c_str(),
+                        d.cite.c_str());
+            }
+        } else {
+            // save expriments
+            std::stringstream ss;
+            req.fetch(ss, cache_dir);
+            std::string line;
+            bool appending = arg["append"];
+            std::ofstream ofs(ofn, appending ? (ofs.out | ofs.app) : ofs.out);
+            while (std::getline(ss, line)) {
+                std::stringstream ssl(line);
+                std::string field;
+                ssl >> field;
+                if (field.size() &&
+                    ((field[0] == '#') || subexp.empty() ||
+                     ((ssl >> field) &&
+                      field.substr(0, subexp.size()) == subexp))) {
+                    ofs << line << std::endl;
+                }
+            }
+        }
+
+    } else {
+        std::cerr << arg[0] << ": script plotting in development" << std::endl;
+        return EXIT_FAILURE;
+    }
+
+    return 0;
+}

cmake/uninstall.cmake

+set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt")
+
+if(NOT EXISTS ${MANIFEST})
+    message(FATAL_ERROR "Cannot find install manifest: '${MANIFEST}'")
+endif()
+
+file(STRINGS ${MANIFEST} files)
+foreach(file ${files})
+    file(GLOB parent_content ${file}/* ${file}/.*)
+    list(LENGTH parent_content parent_len)
+    while(parent_len EQUAL 0)
+	message(STATUS "Removing: '${file}'")
+	file(REMOVE_RECURSE ${file})
+	if(EXISTS ${file})
+	    MESSAGE(STATUS "'${file}' not removed")
+	endif()
+	get_filename_component(file ${file} DIRECTORY)
+	file(GLOB parent_content ${file}/* ${file}/.*)
+	list(LENGTH parent_content parent_len)
+    endwhile()
+
+endforeach(file)

src/crdb.h → include/crdb.h

@@ -7,7 +7,9 @@
 #include <valarray>
 #include <vector>
 
+#ifndef _WITHOUT_ROOT
 #include <TCanvas.h>
+#endif
 
 struct CRCurve {
     std::valarray<double> x, y;
@@ -108,8 +110,7 @@ class CRDataset : public std::vector<std::shared_ptr<CRData>> {
     // TODO: parse as construction
   public:
     void parse_file(const std::string& filename);
-
-    // void parse_remote(const std::string& url);
+    void parse_stream(std::istream& is, const std::string& filename = "[stream]");
 
   public:
     std::shared_ptr<const CRData> find_exact(const std::string& quantity,
@@ -153,6 +154,8 @@ class CRPlot {
     //              const std::array<size_t, 3> columns = {0});
     void DropData(const std::string& label = "");
 
+    // first column is indexed as 1
+    // if columns[0]==0, read first n columns
     void AddCurve(const CRCurve&, const Style&, const std::string& label);
     void AddCurve(const std::string& filename, const Style&,
                   const std::string& label,
@@ -166,5 +169,16 @@ class CRPlot {
     void DropBand(const std::string& label = "");
 
   public:
+#ifndef _WITHOUT_ROOT
     std::shared_ptr<TCanvas> Plot(void) const;
+#endif
+};
+
+struct CRRequest {
+    std::string num = "", den = "";
+    std::string energy_type = "EKN";
+    enum {none = '0', exact = '1', approx = '2'} accuracy = exact;
+    CRDataset fetch(const std::string& cache_dir = "") const;
+    CRDataset& fetch(CRDataset&, const std::string& cache_dir = "") const;
+    std::ostream& fetch(std::ostream&, const std::string cache_dir = "") const;
 };

src/CMakeLists.txt

 cmake_minimum_required(VERSION 2.8)
 
 file(GLOB CRDB_SOURCES *.cc)
-file(GLOB CRDB_HEADERS *.h)
+file(GLOB CRDB_HEADERS ${PROJECT_SOURCE_DIR}/include/crdb.h)
+
+if (${CMAKE_CXX_STANDARD} LESS 17)
+set_source_files_properties(${CRDB_SOURCE_DIR}/src/crrequest.cc PROPERTIES
+    COMPILE_FLAGS "-std=c++1z") # using 1z since the compiler is probably old enough
+endif()
 
 add_library(crdb SHARED ${CRDB_SOURCES})
-target_include_directories(crdb PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+target_include_directories(crdb PUBLIC ${CRDB_SOURCE_DIR}/include)
+target_link_libraries(crdb
+    PRIVATE ${CURL_LIBRARIES}
+    PRIVATE ${ROOT_LIBRRIES}
+    )
 
 set_target_properties(crdb PROPERTIES
     LINKER_LANGUAGE CXX
-    PUBLIC_HEADER "${CRDB_HEADERS}"
-    )
+    PUBLIC_HEADER ${CRDB_HEADERS})
 
 install(TARGETS crdb
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}#/crdb
-    )
+    PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

src/crdata.cc

-#include "crdb.h"
-
 #include "crutil.hh"
-
+#include <crdb.h>
 #include <stdexcept>
 
 double CRData::chi_squared(const std::valarray<double>& y_aligned) const {

src/crdataset.cc

 #include <algorithm>
 #include <fstream>
-#include <iostream>
+// #include <iostream>
 #include <locale>
 #include <memory>
 #include <sstream>
 #include <vector>
 
-#include "crdb.h"
+#include <crdb.h>
 #include "crutil.hh"
 
 #define ADS_BASE "https://ui.adsabs.harvard.edu/abs/"
 
 namespace _CRDATASET {
 
-std::shared_ptr<CRData> parse_data_crd(_CRUTIL::FileParser& fp);
-std::shared_ptr<CRData> parse_data_usine(_CRUTIL::FileParser& fp);
+std::shared_ptr<CRData> parse_data_crd(std::istream&, _CRUTIL::StreamParser&);
+std::shared_ptr<CRData> parse_data_usine(std::istream&, _CRUTIL::StreamParser&);
 
 std::string parse_link(const std::string& hint) {
     if (_CRUTIL::is_substr_i(hint, "http")) {
@@ -32,14 +32,15 @@ std::string parse_link(const std::string& hint) {
 
 }; // namespace _CRDATASET
 
-void CRDataset::parse_file(const std::string& filename) {
+void CRDataset::parse_stream(std::istream& is, const std::string& filename) {
+
     enum { automatic, crd, usine /* , galprop */ } fileformat = automatic;
 
-    _CRUTIL::FileParser fp(filename);
+    _CRUTIL::StreamParser fp(filename);
 
     const auto& line = fp.current_line();
-    fp.next_line(fp.skip::empty);
-    while (fp) {
+    fp.next_line(is, fp.skip::empty);
+    while (is) {
 
         // determine fileformat
         if (fileformat == automatic) {
@@ -49,18 +50,18 @@ void CRDataset::parse_file(const std::string& filename) {
                 // } else if (line.substr(0, 20) == "# Format: USINE code") {
                 fileformat = usine;
             } else { // undetermined, read next line
-                fp.next_line(fp.skip::empty);
+                fp.next_line(is, fp.skip::empty);
                 continue;
             }
         }
 
         switch (fileformat) {
         case crd: {
-            this->push_back(_CRDATASET::parse_data_crd(fp));
+            this->push_back(_CRDATASET::parse_data_crd(is, fp));
             break;
         }
         case usine: {
-            this->push_back(_CRDATASET::parse_data_usine(fp));
+            this->push_back(_CRDATASET::parse_data_usine(is, fp));
             break;
         }
         default: { // no way
@@ -70,13 +71,18 @@ void CRDataset::parse_file(const std::string& filename) {
     }
 }
 
+void CRDataset::parse_file(const std::string& filename) {
+    std::ifstream ifs(filename);
+    parse_stream(ifs, filename);
+}
+
 /** TEMPORARY MACRO **/
 #define _GET_STRING_OPERAND(dest, join) \
     ss >> dest;                         \
     while (ss >> operand)               \
         dest += join + operand;
 
-std::shared_ptr<CRData> _CRDATASET::parse_data_crd(_CRUTIL::FileParser& fp) {
+std::shared_ptr<CRData> _CRDATASET::parse_data_crd(std::istream& is, _CRUTIL::StreamParser& fp) {
     auto pdata = std::make_shared<CRData>();
     auto& data = *pdata;
     std::vector<std::vector<double>> table;
@@ -93,7 +99,7 @@ std::shared_ptr<CRData> _CRDATASET::parse_data_crd(_CRUTIL::FileParser& fp) {
         }
     }
 
-    while (fp.next_line(fp.skip::both)) {
+    while (fp.next_line(is, fp.skip::both), is) {
 
         std::stringstream ss(line);
 
@@ -186,7 +192,7 @@ std::shared_ptr<CRData> _CRDATASET::parse_data_crd(_CRUTIL::FileParser& fp) {
             }
         } else { // empty action for empty line
         }
-    } // while (fp.next_line(fp.skip::both))
+    } // while (fp.next_line(is, fp.skip::both), is)
 
     if (!table.size()) {
         // warn empty data
@@ -240,7 +246,7 @@ std::shared_ptr<CRData> _CRDATASET::parse_data_crd(_CRUTIL::FileParser& fp) {
 
 #undef _GET_STRING_OPERAND
 
-std::shared_ptr<CRData> _CRDATASET::parse_data_usine(_CRUTIL::FileParser& fp) {
+std::shared_ptr<CRData> _CRDATASET::parse_data_usine(std::istream& is, _CRUTIL::StreamParser& fp) {
     auto pdata       = std::make_shared<CRData>();
     auto& data       = *pdata;
     const auto& line = fp.current_line();
@@ -248,9 +254,9 @@ std::shared_ptr<CRData> _CRDATASET::parse_data_usine(_CRUTIL::FileParser& fp) {
     std::vector<std::vector<double>> table;
 
     // not read since the first line may already been read in previous cycle
-    while (fp) {
+    while (is) {
         if (line.empty() || line[0] == '#') {
-            fp.next_line(fp.skip::both);
+            fp.next_line(is, fp.skip::both);
             continue;
         }
 
@@ -332,9 +338,9 @@ std::shared_ptr<CRData> _CRDATASET::parse_data_usine(_CRUTIL::FileParser& fp) {
 
         // repeat for the next line
         // ps.getline(is);
-        fp.next_line(fp.skip::both);
+        fp.next_line(is, fp.skip::both);
 
-    } // while (fp)
+    } // while (is)
 
     // store table into data object
     data._data.resize(table.size() * sizeof(CRData::DataPoint) /
@@ -373,12 +379,14 @@ std::shared_ptr<const CRData> CRDataset::find_exact(const std::string& quantity,
 }
 
 CRData CRData::from_crd(const std::string& filename) {
-    _CRUTIL::FileParser fp(filename);
-    fp.next_line(fp.skip::both);
-    return *_CRDATASET::parse_data_crd(fp);
+    _CRUTIL::StreamParser fp(filename);
+    std::ifstream ifs(filename);
+    fp.next_line(ifs, fp.skip::both);
+    return *_CRDATASET::parse_data_crd(ifs, fp);
 }
 CRData CRData::from_usine(const std::string& filename) {
-    _CRUTIL::FileParser fp(filename);
-    fp.next_line(fp.skip::both);
-    return *_CRDATASET::parse_data_usine(fp);
+    _CRUTIL::StreamParser fp(filename);
+    std::ifstream ifs(filename);
+    fp.next_line(ifs, fp.skip::both);
+    return *_CRDATASET::parse_data_usine(ifs, fp);
 }

src/crplot.cc

-#include "crdb.h"
 #include "crutil.hh"
+#include <crdb.h>
 
 #include <array>
 #include <cmath>

src/crrequest.cc

+#include <curl/curl.h>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#ifndef WITHOUT_ROOT
+#define WITHOUT_ROOT // avoid root error with c++17
+#include <crdb.h>
+#undef WITHOUT_ROOT
+#else
+#include <crdb.h>
+#endif
+
+static size_t WriteCallback(void* contents, size_t size, size_t nmemb,
+                            void* userp) {
+    const size_t realsize = size * nmemb;
+    auto& osv             = *((std::vector<std::ostream*>*)userp);
+    for (auto os : osv) {
+        os->write((char*)contents, realsize);
+    }
+    return realsize;
+}
+
+CRDataset CRRequest::fetch(const std::string& cache_dir) const {
+    CRDataset ds;
+    fetch(ds, cache_dir);
+    return ds;
+}
+
+CRDataset& CRRequest::fetch(CRDataset& ds, const std::string& cache_dir) const {
+    std::stringstream ss;
+    fetch(ss, cache_dir);
+    ds.parse_stream(ss);
+    return ds;
+}
+
+std::ostream& CRRequest::fetch(std::ostream& os,
+                               const std::string cache_dir) const {
+
+    // ensure that numerator is set
+    if (num.empty()) {
+        throw std::logic_error("unspecified numerator CR species");
+    }
+
+    std::string num_used, den_used;
+    {
+        auto islash = num.find('/');
+        if (islash != num.npos) {
+            if (!den.empty()) {
+                throw std::runtime_error(
+                    "duplicated specification of denominator: " + num + '/' +
+                    den);
+            }
+            den_used = num.substr(islash + 1);
+            num_used = num.substr(0, islash);
+        } else {
+            den_used = den;
+            num_used = num;
+        }
+    }
+
+    std::vector<std::ostream*> os_list = {&os};
+    std::ofstream ofs; // must live long enough till writing
+
+    if (!cache_dir.empty()) {
+        // construct cache filename
+        std::string filename =
+            num_used + '_' + (den_used.empty() ? "" : den_used + '_') +
+            energy_type + '_' + static_cast<char>(accuracy) + ".usine";
+
+        std::filesystem::path cache_path(cache_dir);
+        if (!std::filesystem::is_directory(cache_path)) {
+            std::filesystem::create_directories(cache_path); // with exceptions
+        }
+
+        auto file_path = cache_path / filename;
+        if (std::filesystem::exists(file_path)) {
+            // already fetched
+            std::ifstream ifs(file_path.string());
+            return os << ifs.rdbuf();
+        } else {
+            ofs.open(file_path.string());
+            os_list.push_back(&ofs);
+        }
+    }
+
+    // url encode '+'
+    {
+        size_t iplus;
+        while ((iplus = num_used.find('+')) != num_used.npos) {
+            num_used.replace(iplus, 1, "%2B");
+        }
+        while ((iplus = den_used.find('+')) != den_used.npos) {
+            den_used.replace(iplus, 1, "%2B");
+        }
+    }
+
+    // construct url
+    const std::string url =
+        // std::string("http://lpsc.in2p3.fr/crdb/rest.php?format=usine&num=") +
+        std::string(
+            "https://lpsc.in2p3.fr/crdb/_dialog_result.php?format=usine&num=") +
+        num_used + "&den=" + den_used + "&energy_type=" + energy_type +
+        "&combo_level=" + static_cast<char>(accuracy);
+
+    // allocate curl instance
+    CURL* curl = curl_easy_init();
+
+    // specify url to fetch
+    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+    // follow http redirect
+    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
+    // provide user defined context pointer, the list of ostream to write into
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &os_list);
+    // provide user defined writing callback function handling the context pointer
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
+
+    // perform requeset
+    CURLcode code = curl_easy_perform(curl);
+    curl_easy_cleanup(curl);
+
+    if (code) {
+        throw std::runtime_error("curl fetch failed: error code " +
+                                 std::to_string(code));
+    }
+
+    return os;
+}

src/crutil.hh

@@ -3,6 +3,7 @@
 #include <fstream>
 #include <stdexcept>
 #include <string>
+#include <algorithm>
 
 namespace _CRUTIL {
 
@@ -22,6 +23,58 @@ bool is_substr_i(const std::string& sup, const std::string& sub, int pos = 0);
 //     }
 // };
 
+class StreamParser {
+  public:
+    enum skip : uint8_t {
+        none    = 0x00,
+        empty   = 0x01,
+        hashtag = 0x02,
+        both    = 0x03
+    };
+
+  protected:
+    std::string _line, _filename;
+    size_t _line_number;
+
+  public:
+    StreamParser(std::string filename = "[stream]") : _line(""), _filename(filename), _line_number(0) {}
+
+  public:
+    const std::string& current_line(void) const { return _line; }
+    const std::string& filename(void) const { return _filename; }
+    size_t current_line_number(void) const { return _line_number; }
+
+  public:
+    template <typename Ex = std::runtime_error>
+    void panic(const std::string& msg = "unexpected input") const {
+        throw Ex(_filename + ":" + std::to_string(_line_number) + ": " + msg);
+    }
+
+  public:
+    StreamParser& next_line(std::istream& is, skip s = skip::none) {
+        while (std::getline(is, _line)) {
+            ++_line_number;
+            // trim line
+            _line.erase(
+                _line.begin(),
+                std::find_if(_line.begin(), _line.end(), [](unsigned char ch) {
+                    return !std::isspace(ch);
+                }));
+            _line.erase(
+                std::find_if(_line.rbegin(), _line.rend(),
+                             [](unsigned char ch) { return !std::isspace(ch); })
+                    .base(),
+                _line.end());
+
+            if (_line.empty() ? !(s & skip::empty)
+                              : _line[0] != '#' || !(s & skip::hashtag)) {
+                break;
+            }
+        }
+        return *this;
+    }
+};
+
 class FileParser : protected std::ifstream {
   public:
     enum skip : uint8_t {

test/CMakeLists.txt

 cmake_minimum_required(VERSION 2.8)
 
-file(GLOB Test_SOURCES test_read.cc)
 file(GLOB Test_DATA data)
 
-add_executable(test_read ${Test_SOURCES})
+add_executable(test_read "test_read.cc")
 target_link_libraries(test_read crdb ${ROOT_LIBRARIES})
-target_include_directories(test_read PUBLIC ${CMAKE_SOURCE_DIR}/src)
 add_test(
     NAME test:read_crd
     COMMAND bash -c "$<TARGET_FILE:test_read> <(cat ${Test_DATA}/*.crd)")
@@ -15,7 +13,15 @@ add_test(
 
 add_executable(test_plot "test_plot.cc")
 target_link_libraries(test_plot crdb ${ROOT_LIBRARIES})
-target_include_directories(test_plot PUBLIC ${CMAKE_SOURCE_DIR}/src)
 add_test(
     NAME test:plot
     COMMAND test_plot "${Test_DATA}/BC_EKN.usine" "${Test_DATA}/fitted_BC.tsv" "${Test_DATA}/fitted_BC_band.tsv")
+
+add_executable(test_fetch "test_fetch.cc")
+target_link_libraries(test_fetch crdb ${ROOT_LIBRARIES})
+add_test(NAME test:fetch_B_C
+    COMMAND test_fetch "B" "C")
+add_test(NAME test:fetch_p_pe
+    COMMAND test_fetch "e+" "e-+e+")
+add_test(NAME test:fetch_cached
+    COMMAND test_fetch "e+" "e-+e+")

test/test_fetch.cc

+#include <crdb.h>
+#include <iostream>
+
+int main(int argc, char* argv[]) {
+
+    std::string num, den;
+    num = argv[1];
+    den = argc > 2 ? argv[2] : "";
+    CRRequest req{num, den};
+
+    CRDataset ds;
+    req.fetch(ds, "cached_crdata");
+
+    for (auto pd : ds) {
+        auto& d = *pd;
+        std::cout << "exp:\t" << d.experiment << std::endl;
+        std::cout << "time:\t" << d.time << std::endl;
+        std::cout << "cite:\t" << d.cite << std::endl;
+        std::cout << "link:\t" << d.link << std::endl;
+        std::cout << "x:\t" << d.x_quantity << " [ " << d.x_unit << " ]"
+                  << std::endl;
+        std::cout << "y:\t" << d.y_quantity << " [ " << d.y_unit << " ]"
+                  << std::endl;
+        std::cout << "scale:\t" << d.scale << std::endl;
+        std::cout << "hasbin:\t" << d.has_bin << std::endl;
+        std::cout << "skewed:\t" << d.skewed_error << std::endl;
+        std::cout << "rig:\t" << d.use_rigidity << std::endl;
+        std::cout << "phi:\t" << d.phi << std::endl;
+        std::cout << "dist:\t" << d.solar_dist << std::endl;
+        std::cout << std::scientific;
+        std::cout << "#x\t\t#y\t\t#ey1\t\t#ey2\t\t#x_lower\t#x_upper" << std::endl;
+        for (const auto& dp : d) {
+            std::cout << dp.x << '\t' << dp.y << '\t'             //
+                      << dp.ey1 << '\t' << dp.ey2 << '\t'         //
+                      << dp.x_lower << '\t' << dp.x_upper << '\t' //
+                      << std::endl;
+        }
+        std::cout << "total:\t" << d.size() << std::endl;
+    }
+
+    return 0;
+}