1
0
Fork 0

Compare commits

...

3 Commits

Author SHA1 Message Date
pta e07d4ee12d Remove redundant "return"s, add types, rename a variable in my code 2024-05-23 00:58:29 -04:00
pta fb3c3c9199 Replace ugly, inefficient imperative code with functional code
I noticed my algorithm would process the whole array of videos per row
even after finding the target video. To fix it while remaining
imperative, I'd need to break out of a for loop from within a nested if
statement. Maybe it would have worked, but the code would have stayed
ugly and a little hard to reason about.

Using the "filter", "findfirst", and "foreach" higher order functions, I
completely refactored the huge block of nested "for"s and "if"s into a
few functional expressions.

The other notable change is that I'm no longer recording the concurrent
view count on the current live stream, since we're not really interested
in it for our historical analysis purposes, and, being much lower than
the eventual total view count, it would slightly skew calculations
negatively for the current caster.
2024-05-23 00:13:51 -04:00
pta 27e671b16f Place newest wndb script description immediately after the others 2024-05-22 23:59:57 -04:00
4 changed files with 35 additions and 48 deletions

View File

@ -67,12 +67,12 @@ An example of how they should be run can be found in the [`crontab`](crontab) ex
- **bin/wndb-insert.jl** :: This script is meant to be run from cron multiple times per day to insert new rows into the schedule table.
- **bin/wndb-video.jl** :: This script is meant to be run every three hours to add video IDs and view counts to the newest WNL rows in the schedule table.
- **bin/wndb-fix-conflict.jl** :: This script is meant to be run throughout the day to detect and fix schedule changes. If the API response for the schedule conflicts with what's in the database, the API response wins and what was in the database gets moved to the `cancellation` table.
- **bin/wndb-viewcount-update.jl** :: This script is meant to be run twice during every WNL stream to make a last view count update to the stream 240 hours prior.
- **bin/schedule-archive.sh** :: This script is for archiving raw JSON responses from the WeatherNews API. I'm archiving the JSON mostly for redundancy. If `wndb-insert.jl` were to fail, I'd hopefully have JSON data I can use to fix the database after the problem has been fixed.
- **bin/video-archive.sh** :: Archive video JSON responses.
- **bin/mscale-archive.sh** :: Archive mscale JSON responses.
- **bin/mscale-record.sh** :: This script is meant to be run hourly to record the M-scale value from the WeatherNews API.
- **bin/aupay-insert.pl** :: This script scrapes an invidious page for au PAY Market livestreams and inserts new au PAY shows it hasn't seen before. It can be run infrequently like once a week.
- **bin/wndb-viewcount-update.jl** :: This script is meant to be run twice during every WNL stream to make a last view count update to the stream 240 hours prior.
## Web Site

View File

@ -64,7 +64,7 @@ function round_to_segment(zdt::ZonedDateTime)
else
t = Time(23)
end
return ZonedDateTime(d, t, tz"Asia/Tokyo")
ZonedDateTime(d, t, tz"Asia/Tokyo")
end
function massage_fn(zone, leading_zero; t=now())
@ -122,34 +122,28 @@ function get_schedule()
end
"Derive full jst from stream title's date string and WNL segment title"
function wnl_title_to_jst(title)
function wnl_title_to_jst(title::String)
date_re = r"(?<year>[1-9][0-9]{3})[年.](?<month>[0-1]?[0-9])[月.](?<day>[0-3]?[0-9])日?"
d = match(date_re, title)
segment_re = r"モーニング|サンシャイン|コーヒータイム|アフタヌーン|イブニング|ムーン"
s = match(segment_re, title)
h = isnothing(s) ? 23 : HOURS[s.match]
return ZonedDateTime(DateTime(parse(Int64, d[:year]),
parse(Int64, d[:month]),
parse(Int64, d[:day]),
h),
tz"Asia/Tokyo")
ZonedDateTime(DateTime(parse(Int64, d[:year]),
parse(Int64, d[:month]),
parse(Int64, d[:day]),
h),
tz"Asia/Tokyo")
end
"Convert a release timestamp from Unix Epoch to JST ZonedDateTime and round up to the next hour"
function stream_start_to_jst(release)
return round(astimezone(ZonedDateTime(unix2datetime(release), tz"UTC"), tz"Asia/Tokyo"), Dates.Hour(1))
end
"Assume titles with ウェザーニュースLiVE and a full date string belong to regular WNL streams"
function iswnl(title)
return occursin(r"ウェザーニュース[Ll][Ii][Vv][Ee]", title) &&
occursin(r"[1-9][0-9]{3}[年.][0-1]?[0-9][月.][0-3]?[0-9]日?", title)
function stream_start_to_jst(release::Int)
round(astimezone(ZonedDateTime(unix2datetime(release), tz"UTC"), tz"Asia/Tokyo"), Dates.Hour(1))
end
"Make a final view count update after ten days."
function update_old_video(db)
一昨= round_to_segment(now(tz"Asia/Tokyo") - Day(10))
row = DB.find_schedule_by_jst(db, string(一昨))
先日 = round_to_segment(now(tz"Asia/Tokyo") - Day(10))
row = DB.find_schedule_by_jst(db, string(先日))
vid = API.video_ids(row[:video_id])
DB.update_schedule_with_viewcount(db, vid[:view_count], row[:id])
end
@ -164,31 +158,20 @@ julia> WeatherNews.update_schedule_with_latest_videos(db)
"""
function update_schedule_with_latest_videos(db)
schedule = DB.find_upcoming_WNL_schedule(db)
vids = API.video_ids()
for row in eachrow(schedule)
schedjst = row[:jst] # could be halfway into segment if shared segment
for v in vids
if iswnl(v[:title])
vidjst =
if v[:live_status] == "is_upcoming"
stream_start_to_jst(v[:release_timestamp])
else
wnl_title_to_jst(v[:title])
end
if round_to_segment(schedjst) == vidjst
views =
if v[:live_status] == "was_live"
v[:view_count]
elseif v[:live_status] == "is_live"
v[:concurrent_view_count]
else # "is_upcoming"
missing
end
DB.update_schedule_with_video(db, v[:id], views, row[:id])
end
end
videos = API.video_ids()
videojst = v ->
if v[:live_status] == "is_upcoming"
stream_start_to_jst(v[:release_timestamp])
else
wnl_title_to_jst(v[:title])
end
end
foreach(row -> let
index = findfirst(v -> round_to_segment(row[:jst]) == videojst(v), videos)
isnothing(index) && return
views = videos[index][:live_status] == "was_live" ? videos[index][:view_count] : missing
DB.update_schedule_with_video(db, videos[index][:id], views, row[:id])
end,
eachrow(schedule))
end
"""

View File

@ -36,13 +36,17 @@ function video_ids()
--dump-single-json
--playlist-items 1:30
https://www.youtube.com/weathernews/streams`
playlist = readchomp(cmd) |> JSON3.read
return playlist[:"entries"]
# Filter out non-WNL streams.
# Assumes videos with ウェザーニュースLiVE and a full date string in the title belong to regular WNL streams.
wnl_only = videos -> filter(v -> occursin(r"ウェザーニュース[Ll][Ii][Vv][Ee]", v[:title]) &&
occursin(r"[1-9][0-9]{3}[年.][0-1]?[0-9][月.][0-3]?[0-9]日?", v[:title]),
videos)
readchomp(cmd) |> JSON3.read |> x -> x[:entries] |> wnl_only
end
function video_ids(id::String)
cmd = `yt-dlp --extractor-args "youtube:player-client=web"
--dump-single-json
"https://www.youtube.com/watch?v=$(id)"`
return readchomp(cmd) |> JSON3.read
readchomp(cmd) |> JSON3.read
end

View File

@ -50,7 +50,7 @@ function find_upcoming_WNL_schedule(db)
sql_select = "SELECT id, jst FROM schedule WHERE segment_id IS NOT 8 ORDER BY jst DESC limit 12;"
s = DBInterface.execute(db, sql_select) |> DataFrame
s.jst = jst2zdt.(s.jst)
return s
s
end
"""
@ -106,7 +106,7 @@ end
function update_schedule_with_video(db, video_id::String, view_count::Union{Int,Missing}, id::Int)
try
sql_update = "UPDATE schedule SET video_id = ?, view_count = ? WHERE id = ?;"
return DBInterface.execute(db, sql_update, [video_id, view_count, id])
DBInterface.execute(db, sql_update, [video_id, view_count, id])
catch e
@debug e
return e
@ -117,7 +117,7 @@ end
function update_schedule_with_viewcount(db, view_count::Int, id::Int)
try
sql_update = "UPDATE schedule SET view_count = ? WHERE id = ?;"
return DBInterface.execute(db, sql_update, [view_count, id])
DBInterface.execute(db, sql_update, [view_count, id])
catch e
@debug e
return e