forked from gg/WeatherNews.jl
Compare commits
3 Commits
cb105176b2
...
e07d4ee12d
Author | SHA1 | Date |
---|---|---|
pta | e07d4ee12d | |
pta | fb3c3c9199 | |
pta | 27e671b16f |
|
@ -67,12 +67,12 @@ An example of how they should be run can be found in the [`crontab`](crontab) ex
|
|||
- **bin/wndb-insert.jl** :: This script is meant to be run from cron multiple times per day to insert new rows into the schedule table.
|
||||
- **bin/wndb-video.jl** :: This script is meant to be run every three hours to add video IDs and view counts to the newest WNL rows in the schedule table.
|
||||
- **bin/wndb-fix-conflict.jl** :: This script is meant to be run throughout the day to detect and fix schedule changes. If the API response for the schedule conflicts with what's in the database, the API response wins and what was in the database gets moved to the `cancellation` table.
|
||||
- **bin/wndb-viewcount-update.jl** :: This script is meant to be run twice during every WNL stream to make a last view count update to the stream 240 hours prior.
|
||||
- **bin/schedule-archive.sh** :: This script is for archiving raw JSON responses from the WeatherNews API. I'm archiving the JSON mostly for redundancy. If `wndb-insert.jl` were to fail, I'd hopefully have JSON data I can use to fix the database after the problem has been fixed.
|
||||
- **bin/video-archive.sh** :: Archive video JSON responses.
|
||||
- **bin/mscale-archive.sh** :: Archive mscale JSON responses.
|
||||
- **bin/mscale-record.sh** :: This script is meant to be run hourly to record the M-scale value from the WeatherNews API.
|
||||
- **bin/aupay-insert.pl** :: This script scrapes an invidious page for au PAY Market livestreams and inserts new au PAY shows it hasn't seen before. It can be run infrequently like once a week.
|
||||
- **bin/wndb-viewcount-update.jl** :: This script is meant to be run twice during every WNL stream to make a last view count update to the stream 240 hours prior.
|
||||
|
||||
## Web Site
|
||||
|
||||
|
|
|
@ -64,7 +64,7 @@ function round_to_segment(zdt::ZonedDateTime)
|
|||
else
|
||||
t = Time(23)
|
||||
end
|
||||
return ZonedDateTime(d, t, tz"Asia/Tokyo")
|
||||
ZonedDateTime(d, t, tz"Asia/Tokyo")
|
||||
end
|
||||
|
||||
function massage_fn(zone, leading_zero; t=now())
|
||||
|
@ -122,34 +122,28 @@ function get_schedule()
|
|||
end
|
||||
|
||||
"Derive full jst from stream title's date string and WNL segment title"
|
||||
function wnl_title_to_jst(title)
|
||||
function wnl_title_to_jst(title::String)
|
||||
date_re = r"(?<year>[1-9][0-9]{3})[年.](?<month>[0-1]?[0-9])[月.](?<day>[0-3]?[0-9])日?"
|
||||
d = match(date_re, title)
|
||||
segment_re = r"モーニング|サンシャイン|コーヒータイム|アフタヌーン|イブニング|ムーン"
|
||||
s = match(segment_re, title)
|
||||
h = isnothing(s) ? 23 : HOURS[s.match]
|
||||
return ZonedDateTime(DateTime(parse(Int64, d[:year]),
|
||||
parse(Int64, d[:month]),
|
||||
parse(Int64, d[:day]),
|
||||
h),
|
||||
tz"Asia/Tokyo")
|
||||
ZonedDateTime(DateTime(parse(Int64, d[:year]),
|
||||
parse(Int64, d[:month]),
|
||||
parse(Int64, d[:day]),
|
||||
h),
|
||||
tz"Asia/Tokyo")
|
||||
end
|
||||
|
||||
"Convert a release timestamp from Unix Epoch to JST ZonedDateTime and round up to the next hour"
|
||||
function stream_start_to_jst(release)
|
||||
return round(astimezone(ZonedDateTime(unix2datetime(release), tz"UTC"), tz"Asia/Tokyo"), Dates.Hour(1))
|
||||
end
|
||||
|
||||
"Assume titles with ウェザーニュースLiVE and a full date string belong to regular WNL streams"
|
||||
function iswnl(title)
|
||||
return occursin(r"ウェザーニュース[Ll][Ii][Vv][Ee]", title) &&
|
||||
occursin(r"[1-9][0-9]{3}[年.][0-1]?[0-9][月.][0-3]?[0-9]日?", title)
|
||||
function stream_start_to_jst(release::Int)
|
||||
round(astimezone(ZonedDateTime(unix2datetime(release), tz"UTC"), tz"Asia/Tokyo"), Dates.Hour(1))
|
||||
end
|
||||
|
||||
"Make a final view count update after ten days."
|
||||
function update_old_video(db)
|
||||
一昨日 = round_to_segment(now(tz"Asia/Tokyo") - Day(10))
|
||||
row = DB.find_schedule_by_jst(db, string(一昨日))
|
||||
先日 = round_to_segment(now(tz"Asia/Tokyo") - Day(10))
|
||||
row = DB.find_schedule_by_jst(db, string(先日))
|
||||
vid = API.video_ids(row[:video_id])
|
||||
DB.update_schedule_with_viewcount(db, vid[:view_count], row[:id])
|
||||
end
|
||||
|
@ -164,31 +158,20 @@ julia> WeatherNews.update_schedule_with_latest_videos(db)
|
|||
"""
|
||||
function update_schedule_with_latest_videos(db)
|
||||
schedule = DB.find_upcoming_WNL_schedule(db)
|
||||
vids = API.video_ids()
|
||||
for row in eachrow(schedule)
|
||||
schedjst = row[:jst] # could be halfway into segment if shared segment
|
||||
for v in vids
|
||||
if iswnl(v[:title])
|
||||
vidjst =
|
||||
if v[:live_status] == "is_upcoming"
|
||||
stream_start_to_jst(v[:release_timestamp])
|
||||
else
|
||||
wnl_title_to_jst(v[:title])
|
||||
end
|
||||
if round_to_segment(schedjst) == vidjst
|
||||
views =
|
||||
if v[:live_status] == "was_live"
|
||||
v[:view_count]
|
||||
elseif v[:live_status] == "is_live"
|
||||
v[:concurrent_view_count]
|
||||
else # "is_upcoming"
|
||||
missing
|
||||
end
|
||||
DB.update_schedule_with_video(db, v[:id], views, row[:id])
|
||||
end
|
||||
end
|
||||
videos = API.video_ids()
|
||||
videojst = v ->
|
||||
if v[:live_status] == "is_upcoming"
|
||||
stream_start_to_jst(v[:release_timestamp])
|
||||
else
|
||||
wnl_title_to_jst(v[:title])
|
||||
end
|
||||
end
|
||||
foreach(row -> let
|
||||
index = findfirst(v -> round_to_segment(row[:jst]) == videojst(v), videos)
|
||||
isnothing(index) && return
|
||||
views = videos[index][:live_status] == "was_live" ? videos[index][:view_count] : missing
|
||||
DB.update_schedule_with_video(db, videos[index][:id], views, row[:id])
|
||||
end,
|
||||
eachrow(schedule))
|
||||
end
|
||||
|
||||
"""
|
||||
|
|
10
src/api.jl
10
src/api.jl
|
@ -36,13 +36,17 @@ function video_ids()
|
|||
--dump-single-json
|
||||
--playlist-items 1:30
|
||||
https://www.youtube.com/weathernews/streams`
|
||||
playlist = readchomp(cmd) |> JSON3.read
|
||||
return playlist[:"entries"]
|
||||
# Filter out non-WNL streams.
|
||||
# Assumes videos with ウェザーニュースLiVE and a full date string in the title belong to regular WNL streams.
|
||||
wnl_only = videos -> filter(v -> occursin(r"ウェザーニュース[Ll][Ii][Vv][Ee]", v[:title]) &&
|
||||
occursin(r"[1-9][0-9]{3}[年.][0-1]?[0-9][月.][0-3]?[0-9]日?", v[:title]),
|
||||
videos)
|
||||
readchomp(cmd) |> JSON3.read |> x -> x[:entries] |> wnl_only
|
||||
end
|
||||
|
||||
function video_ids(id::String)
|
||||
cmd = `yt-dlp --extractor-args "youtube:player-client=web"
|
||||
--dump-single-json
|
||||
"https://www.youtube.com/watch?v=$(id)"`
|
||||
return readchomp(cmd) |> JSON3.read
|
||||
readchomp(cmd) |> JSON3.read
|
||||
end
|
||||
|
|
|
@ -50,7 +50,7 @@ function find_upcoming_WNL_schedule(db)
|
|||
sql_select = "SELECT id, jst FROM schedule WHERE segment_id IS NOT 8 ORDER BY jst DESC limit 12;"
|
||||
s = DBInterface.execute(db, sql_select) |> DataFrame
|
||||
s.jst = jst2zdt.(s.jst)
|
||||
return s
|
||||
s
|
||||
end
|
||||
|
||||
"""
|
||||
|
@ -106,7 +106,7 @@ end
|
|||
function update_schedule_with_video(db, video_id::String, view_count::Union{Int,Missing}, id::Int)
|
||||
try
|
||||
sql_update = "UPDATE schedule SET video_id = ?, view_count = ? WHERE id = ?;"
|
||||
return DBInterface.execute(db, sql_update, [video_id, view_count, id])
|
||||
DBInterface.execute(db, sql_update, [video_id, view_count, id])
|
||||
catch e
|
||||
@debug e
|
||||
return e
|
||||
|
@ -117,7 +117,7 @@ end
|
|||
function update_schedule_with_viewcount(db, view_count::Int, id::Int)
|
||||
try
|
||||
sql_update = "UPDATE schedule SET view_count = ? WHERE id = ?;"
|
||||
return DBInterface.execute(db, sql_update, [view_count, id])
|
||||
DBInterface.execute(db, sql_update, [view_count, id])
|
||||
catch e
|
||||
@debug e
|
||||
return e
|
||||
|
|
Loading…
Reference in New Issue