|
|
|
@ -6,6 +6,8 @@ use hyper::Uri; |
|
|
|
|
|
|
|
use chrono::naive::NaiveDate;
|
|
|
|
use chrono::Datelike;
|
|
|
|
use chrono::Utc;
|
|
|
|
use chrono::FixedOffset;
|
|
|
|
|
|
|
|
use hyper_tls::HttpsConnector;
|
|
|
|
|
|
|
|
@ -14,49 +16,129 @@ use scraper::Selector; |
|
|
|
use scraper::ElementRef;
|
|
|
|
use scraper::Node;
|
|
|
|
|
|
|
|
use ego_tree::NodeRef;
|
|
|
|
|
|
|
|
use tokio::runtime::Runtime;
|
|
|
|
use json::JsonValue;
|
|
|
|
use json::stringify;
|
|
|
|
use json::object;
|
|
|
|
|
|
|
|
use clap::command;
|
|
|
|
use clap::Command;
|
|
|
|
use clap::Arg;
|
|
|
|
use clap::ArgAction;
|
|
|
|
use clap::ArgMatches;
|
|
|
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
struct Question {
|
|
|
|
question: String,
|
|
|
|
answer: String,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Question {
|
|
|
|
fn to_json(&self) -> JsonValue {
|
|
|
|
//TODO: fix this syntax to match the actual cards
|
|
|
|
struct Anki {
|
|
|
|
uri: Uri,
|
|
|
|
deck_id: u64,
|
|
|
|
model_id: u64,
|
|
|
|
question_field: String,
|
|
|
|
answer_field: String,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Anki {
|
|
|
|
fn new() -> Self {
|
|
|
|
Anki {
|
|
|
|
uri: Uri::from_static("http://localhost:8765"),
|
|
|
|
deck_id: 1683334432853, //TODO: use these to compute name
|
|
|
|
model_id: 1679827221996, //TODO: use these to compute name
|
|
|
|
question_field: String::from("表面"),
|
|
|
|
answer_field: String::from("裏面"),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn command(&self, action: &str, params: JsonValue) -> Result<JsonValue, String> {
|
|
|
|
let payload = object!{
|
|
|
|
//action: "addNotes",
|
|
|
|
action: action,
|
|
|
|
version: 6,
|
|
|
|
params: params,
|
|
|
|
};
|
|
|
|
let client = Client::new();
|
|
|
|
let request = Request::builder()
|
|
|
|
.uri(self.uri.clone())
|
|
|
|
.body(Body::from(json::stringify(payload))).unwrap();
|
|
|
|
let response = client.request(request).await.unwrap();
|
|
|
|
let body_bytes = hyper::body::to_bytes(response).await.unwrap().to_vec();
|
|
|
|
let payload = std::str::from_utf8(&body_bytes).map(json::parse).unwrap().unwrap();
|
|
|
|
//println!("{:?}", payload);
|
|
|
|
Ok(payload["result"].clone())
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn list_decks(&self) -> Vec<(String, u64)> {
|
|
|
|
self.command("deckNamesAndIds", object!{}).await.unwrap().entries()
|
|
|
|
.map(|(name, id)| (name.to_string(), id.as_u64().unwrap())).collect()
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn list_models(&self) -> Vec<(String, u64)> {
|
|
|
|
self.command("modelNamesAndIds", object!{}).await.unwrap().entries()
|
|
|
|
.map(|(name, id)| (name.to_string(), id.as_u64().unwrap())).collect()
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn model_fields(&self, model: &str) -> Vec<String> {
|
|
|
|
self.command("modelFieldNames", object!{modelName: model}).await.unwrap().members().map(JsonValue::to_string).collect()
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn add_notes(&self, questions: Vec<Question>) -> Result<JsonValue, String> {
|
|
|
|
let deck_name = lookup_name(self.deck_id, &self.list_decks().await);
|
|
|
|
let model_name = lookup_name(self.model_id, &self.list_models().await);
|
|
|
|
let jsonify_question = |question: &Question| {
|
|
|
|
let mut fields = JsonValue::new_object();
|
|
|
|
fields[self.question_field.clone()] = JsonValue::String(question.question.clone());
|
|
|
|
fields[self.answer_field.clone()] = JsonValue::String(question.answer.clone());
|
|
|
|
object!{
|
|
|
|
question: self.question.clone(),
|
|
|
|
answer: self.answer.clone(),
|
|
|
|
deckName: deck_name.clone(),
|
|
|
|
modelName: model_name.clone(),
|
|
|
|
fields: fields,
|
|
|
|
tags: ["毎日漢字"],
|
|
|
|
}
|
|
|
|
};
|
|
|
|
let json_questions = questions.iter().map(jsonify_question).collect::<Vec<_>>();
|
|
|
|
println!("Questions {:?}", json_questions);
|
|
|
|
self.command("addNotes", object!{notes: json_questions}).await
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
struct Anki {
|
|
|
|
uri: Uri,
|
|
|
|
deck_id: String,
|
|
|
|
fn lookup_name(lookup_id: u64, names: &Vec<(String, u64)>) -> Option<String> {
|
|
|
|
for (name, id) in names {
|
|
|
|
if *id == lookup_id {
|
|
|
|
return Some(name.to_string())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
None
|
|
|
|
}
|
|
|
|
|
|
|
|
fn main() {
|
|
|
|
//ensure datasets are loaded
|
|
|
|
// download pages
|
|
|
|
// parse questions
|
|
|
|
//ensure questions are in anki
|
|
|
|
println!("Hello, world!");
|
|
|
|
let matches = command!()
|
|
|
|
.subcommand_required(true)
|
|
|
|
.subcommand(
|
|
|
|
Command::new("import")
|
|
|
|
.about("download data to Anki")
|
|
|
|
.arg(Arg::new("force").short('f').action(ArgAction::SetTrue).help("actually run the import"))
|
|
|
|
)
|
|
|
|
.get_matches();
|
|
|
|
|
|
|
|
match matches.subcommand() {
|
|
|
|
Some(("import", sub_matches)) => cmd_import(sub_matches),
|
|
|
|
_ => unreachable!("No subcommand provided!")
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
fn cmd_import(args: &ArgMatches) {
|
|
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
// the window of available data is like today - 9 days
|
|
|
|
let today = Utc::now().with_timezone(&FixedOffset::east_opt(9 * 3600).unwrap()).date_naive();
|
|
|
|
// TODO: cache downloaded pages
|
|
|
|
// TODO: have an overall cacheing strategy
|
|
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
// the window of available data is like today - 15 days or something
|
|
|
|
let for_date = NaiveDate::from_ymd_opt(2023, 7, 27).unwrap(); // TODO: replace with today
|
|
|
|
//let questions = rt.block_on(scrape_questions(for_date));
|
|
|
|
let anki = Anki{ uri: Uri::from_static("http://localhost:8765"), deck_id: "lol".to_string()};
|
|
|
|
//println!("Questions: {:?}", questions);
|
|
|
|
rt.block_on(ensure_in_anki(&anki, vec![]));
|
|
|
|
let questions = rt.block_on(scrape_questions(today)).unwrap();
|
|
|
|
println!("Questions: {:?}", questions);
|
|
|
|
let anki = Anki::new();
|
|
|
|
rt.block_on(ensure_in_anki(&anki, questions));
|
|
|
|
//rt.block_on(ensure_in_anki(&anki, questions.unwrap()));
|
|
|
|
}
|
|
|
|
|
|
|
|
@ -89,7 +171,7 @@ fn parse_question(element: ElementRef) -> Option<Question> { |
|
|
|
.take_while(|x| ElementRef::wrap(*x) != question_end)
|
|
|
|
.map(|node| match node.value() {
|
|
|
|
Node::Text(text) => String::from(text.text.trim()),
|
|
|
|
Node::Element(_element) => format!("<em>{}</em>", node.first_child().unwrap().value().as_text().unwrap().text.trim()),
|
|
|
|
Node::Element(_element) => format!("<u>{}</u>", node.first_child().unwrap().value().as_text().unwrap().text.trim()),
|
|
|
|
_ => panic!("fresh peach heart shower"),
|
|
|
|
})
|
|
|
|
.collect::<Vec<_>>()
|
|
|
|
@ -114,25 +196,7 @@ fn not_in_anki(_question: &Question) -> bool { |
|
|
|
}
|
|
|
|
|
|
|
|
async fn upload_to_anki(anki: &Anki, questions: Vec<Question>) -> Result<(), String> {
|
|
|
|
let notes = JsonValue::Array(questions.iter().map(Question::to_json).collect());
|
|
|
|
let payload = object!{
|
|
|
|
//action: "addNotes",
|
|
|
|
action: "deckNamesAndIds",
|
|
|
|
version: 6,
|
|
|
|
params: {
|
|
|
|
//notes: notes,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
let client = Client::new();
|
|
|
|
let request = Request::builder()
|
|
|
|
.uri(Uri::from_static("http://localhost:8765"))
|
|
|
|
.body(Body::from(json::stringify(payload))).unwrap();
|
|
|
|
let response = client.request(request).await.unwrap();
|
|
|
|
let body_bytes = hyper::body::to_bytes(response).await.unwrap().to_vec();
|
|
|
|
let payload = json::parse(std::str::from_utf8(&body_bytes).unwrap()).unwrap();
|
|
|
|
println!("Data: {:?}", payload["result"]);
|
|
|
|
for (deckName, id) in payload["result"].entries() {
|
|
|
|
println!("Deck: {} {}", deckName, id.as_u64().unwrap());
|
|
|
|
}
|
|
|
|
let result = anki.add_notes(questions).await;
|
|
|
|
println!("Result: {:?}", result);
|
|
|
|
Ok(())
|
|
|
|
}
|
