/*
Copyright 2018-2020 eomanis
This file is part of getgarfield.
getgarfield is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License version 3 as
published by the Free Software Foundation.
getgarfield is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with getgarfield. If not, see <http://www.gnu.org/licenses/>.
*/
package getgarfield;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.time.LocalDate;
import java.time.Year;
import java.time.YearMonth;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAdjusters;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Stream;
/**
* <p>
* Downloads all Garfield comic strips into a given target directory.
* </p>
* <p>
* Also creates two XHTML web sites that present the comic strips by month, one "offline" web site that displays the downloaded strips and another "online" web
* site that uses the original source URLs where the strips are downloaded from.
* </p>
* <p>
* Skips comic strips that are already present; running it again on the same target directory will just re-create the web sites and download what new comic
* strips are missing.
* </p>
* <p>
* As of May 2020 the downloaded comic strips amount to about 1.8 GiB.
* </p>
*
* @author eomanis
*/
public class GetGarfield implements Runnable {
public static final String APPLICATION_NAME = "getgarfield";
public static final String APPLICATION_DESCRIPTION = "Java application that downloads all Garfield comic strips";
// Semantic versioning
public static final int VERSION_MAJOR = 0;
public static final int VERSION_MINOR = 2;
public static final int VERSION_PATCH = 2;
public static final String VERSION_LABEL = "";
public static final String VERSION = VERSION_MAJOR + "." + VERSION_MINOR + "." + VERSION_PATCH
+ (!VERSION_LABEL.isEmpty() ? ("-" + VERSION_LABEL) : "");
// Constants that can be fiddled with during development
// Important: Set all of them to false for releases!
// ===========================================================================================
private static final boolean DEBUG = false; // Enable debug messages
private static final boolean NO_DOWNLOAD = false; // Do not actually download any comic strips
// Constants that control various aspects of the application
// Here is where you should start looking if you want to tweak something
// ===========================================================================================
// The Locale used by the DateTimeFormatters responsible for file names and the years/months/days
// shown or otherwise used in the generated web sites
private static final Locale LOCALE_DATE_TIME_FORMATTERS = new Locale( "en" );
// How many concurrent downloads should be used
private static final int CONCURRENT_DOWNLOADS_COUNT = 8;
// How many days to scan backwards from today when determining the latest comic strip before
// resorting to a binary search between the most recent known comic strip and today
private static final int LATEST_COMIC_STRIP_QUICK_SCAN_LAST_DAYS = 3;
// The suffix to use for temporary files
private static final String TEMP_FILE_NAME_SUFFIX = ".tmp";
// How many days into the future, at least, the "online" web site should be created
private static final int WEB_SITE_ONLINE_GENERATE_MIN_DAYS_INTO_FUTURE = 14;
// The name of the sub-directory into which to download the comic strips
private static final String SUBDIR_COMIC_STRIPS_NAME = "comic.strips";
// Whether to skip re-creation of existing "comic strips of month" pages when it can be assumed that they contain the whole month
private static final boolean AUTO_SKIP_FULL_MONTH_PAGES = true;
// Date format for a day: File name of a downloaded comic strip
private static final DateTimeFormatter FORMATTER_DAY_COMIC_STRIP_FILE_NAME = DateTimeFormatter.ofPattern( "'garfield.'uuuu-MM-dd-EEE'.gif'",
LOCALE_DATE_TIME_FORMATTERS );
// Date format for a year: As entry in the "table of contents" page
private static final DateTimeFormatter FORMATTER_YEAR_PAGE_TOC = DateTimeFormatter.ofPattern( "uuuu", LOCALE_DATE_TIME_FORMATTERS );
// Date format for a month: As title for a "comic strips of month" page
private static final DateTimeFormatter FORMATTER_MONTH_PAGE_TITLE = DateTimeFormatter.ofPattern( "uuuu-MM MMMM", LOCALE_DATE_TIME_FORMATTERS );
// Date format for a month: As entry in the "table of contents" page
private static final DateTimeFormatter FORMATTER_MONTH_PAGE_TOC = DateTimeFormatter.ofPattern( "MMM", LOCALE_DATE_TIME_FORMATTERS );
// Date format for a month: In the navigation/controls of a "comic strips of month" page
private static final DateTimeFormatter FORMATTER_MONTH_PAGE_NAVIGATION = DateTimeFormatter.ofPattern( "uuuu-MM' 'MMM", LOCALE_DATE_TIME_FORMATTERS );
// Date format for a month: File name for a "comic strips of month" page
private static final DateTimeFormatter FORMATTER_MONTH_PAGE_FILE_NAME = DateTimeFormatter.ofPattern( "uuuu-MM'.xhtml'", LOCALE_DATE_TIME_FORMATTERS );
// Date format for a day: Alternative text for a comic strip in a "comic strips of month" page
private static final DateTimeFormatter FORMATTER_DAY_PAGE_COMIC_STRIP_ALT_TEXT = DateTimeFormatter.ofPattern( "uuuu-MM-dd EEE",
LOCALE_DATE_TIME_FORMATTERS );
// Date format for a day: Text for a known missing comic strip in a "comic strips of month" page
private static final DateTimeFormatter FORMATTER_DAY_PAGE_COMIC_STRIP_MISSING = DateTimeFormatter.ofPattern(
"uuuu-MM-dd EEE': No comic strip published for this day'", LOCALE_DATE_TIME_FORMATTERS );
// Date format for a day: XML node ID of a comic strip in a "comic strips of month" page (day of month)
private static final DateTimeFormatter FORMATTER_DAY_PAGE_COMIC_STRIP_ID = DateTimeFormatter.ofPattern( "'day'd", LOCALE_DATE_TIME_FORMATTERS );
// The file name that should be used for the generated web sites' Cascading Style Sheet (CSS)
private static final String STYLES_FILE_NAME = "styles.css";
// Other constants that you should only change if you know what you are doing
// ===========================================================================================
// The very first comic strip
private static final LocalDate FIRST_COMIC_STRIP_DATE = LocalDate.parse( "1978-06-19" );
// The most recent comic strip that is known to exist
private static final LocalDate LATEST_COMIC_STRIP_DATE_KNOWN = LocalDate.parse( "2020-07-04" );
// Days for which no comic strip was published
private static final Set<LocalDate> KNOWN_MISSING_COMIC_STRIPS = Collections.unmodifiableSet( new HashSet<>( Arrays.asList( new LocalDate[] {
// No missing comic strips as of 2020-07-04
} ) ) );
// The time zone in which the comic strip is being published
private static final ZoneId TIME_ZONE_COMIC_STRIP = ZoneId.of( "US/Eastern" );
// The web site templates' and generated sites' character set
private static final String WEB_SITE_TEMPLATE_ENCODING_NAME = "UTF-8";
// Various placeholder strings in the web site templates
// Any text line that ends with such a string is being replaced with the respective content when generating the web sites
private static final String WEB_SITE_TEMPLATE_PLACEHOLDER_TITLE = "<title>#TITLE#</title>";
private static final String WEB_SITE_TEMPLATE_PLACEHOLDER_STYLES = "<link rel=\"stylesheet\" type=\"text/css\" href=\"styles.css\" />";
private static final String WEB_SITE_TEMPLATE_PLACEHOLDER_TOC = "<p>#TABLE_OF_CONTENTS#</p>";
private static final String WEB_SITE_TEMPLATE_PLACEHOLDER_NAVIGATION = "<p>#NAVIGATION#</p>";
private static final String WEB_SITE_TEMPLATE_PLACEHOLDER_HEADING = "<p>#HEADING#</p>";
private static final String WEB_SITE_TEMPLATE_PLACEHOLDER_COMICS = "<p>#COMICS#</p>";
// The threads' byte buffers that are used for copying or downloading data
private static final ThreadLocal<ByteBuffer> TL_BYTE_BUFFER = ThreadLocal.withInitial( () -> ByteBuffer.allocate( 64 * 1024 ) );
// Where the comic strips should be downloaded from (URL)
private static final DateTimeFormatter FORMATTER_COMIC_STRIP_URL = DateTimeFormatter.ofPattern(
"'https://d1ejxu6vysztl5.cloudfront.net/comics/garfield/'uuuu'/'uuuu-MM-dd'.gif'", new Locale( "en" ) );
// Alternate comic strip URL formatter (legacy, low-resolution and apparently unmaintained comic strips):
// DateTimeFormatter.ofPattern( "'http://images.ucomics.com/comics/ga/'uuuu'/ga'uuMMdd'.gif'", new Locale( "en" ) );
/**
* <p>
* The target directory where everything is written/downloaded to, possibly into {@link #subdirComicStrips sub-directories}
* </p>
*/
private final Path targetDir;
/**
* <p>
* The greatest day, including, up to which to download the comic strips
* </p>
*/
private final LocalDate greatestDateIncl;
/**
* <p>
* The sub-directory inside the {@link #targetDir target directory} into which to download the comic strips
* </p>
*/
private final Path subdirComicStrips;
public static void main( String[] args ) {
printlnInfo( "This is " + APPLICATION_NAME + " " + VERSION );
printlnDebug( "Debug messages enabled by constant" );
if (NO_DOWNLOAD) {
printlnWarning( "Downloads disabled by constant" );
}
if (args.length != 1) {
if (args.length == 0) {
printlnInfo( APPLICATION_DESCRIPTION );
}
printlnError( "Single argument required: Target directory" );
System.exit( 1 );
}
try {
new GetGarfield( Paths.get( args[0] ), getLatestComicStripDate() ).run();
} catch (RuntimeException e) {
printlnError( e.getLocalizedMessage() );
System.exit( 1 );
}
}
public GetGarfield( Path targetDir, LocalDate greatestDateIncl ) {
super();
this.targetDir = targetDir;
this.greatestDateIncl = greatestDateIncl;
this.subdirComicStrips = targetDir.resolve( SUBDIR_COMIC_STRIPS_NAME );
}
@Override
public void run() {
try {
downloadComicStripsAndCreateWebSites();
} catch (@SuppressWarnings("unused") InterruptedException e) { // The downloads shutdown hook is handling this
Thread.currentThread().interrupt();
} catch (IOException e) {
throw new RuntimeException( e );
}
}
/**
* @throws InterruptedException
* if {@link ExecutorService#awaitTermination(long, TimeUnit) waiting} for the concurrent downloads to finish is interrupted
* @throws IOException
* if creating the target directory structure or generating the web sites failed
*/
private void downloadComicStripsAndCreateWebSites() throws InterruptedException, IOException {
printlnInfo( "Target directory: \"" + targetDir + "\"" );
printlnInfo( "Getting comic strips up to including " + greatestDateIncl );
Files.createDirectories( subdirComicStrips );
// Use a fixed thread pool that gets its jobs from a central queue
// This causes the files to be downloaded in an approximately chronological order
// With all the I/O going on queue contention is not an issue here
printlnInfo( "Starting " + CONCURRENT_DOWNLOADS_COUNT + " download worker threads" );
ExecutorService executorService = Executors.newFixedThreadPool( CONCURRENT_DOWNLOADS_COUNT );
Runtime.getRuntime().addShutdownHook( createDownloadsShutdownHook( executorService ) );
Iterator<LocalDate> daysToDownload = createStreamOfDays( greatestDateIncl ).filter( date -> !Files.exists( getComicStripFile( date ) ) ).iterator();
// Start off by enqueueing a single download Runnable
// Each download Runnable will immediately enqueue another such Runnable when it is run, and the resulting avalanche effect will saturate all download threads
// That way we avoid creating all Runnables up front, there are quite many of them for an initial run
// It's just a shame that an ExecutorService cannot just consume an Iterable or Stream of Runnables
enqueueNextDownloadRunnable( executorService, daysToDownload );
// Create some web pages while stuff is being downloaded
createWebSite( "index.online", Integer.valueOf( WEB_SITE_ONLINE_GENERATE_MIN_DAYS_INTO_FUTURE ), FORMATTER_COMIC_STRIP_URL::format );
createWebSite( "index", (Integer) null, date -> "../" + SUBDIR_COMIC_STRIPS_NAME + "/" + FORMATTER_DAY_COMIC_STRIP_FILE_NAME.format( date ) );
// Wait until all downloads have completed
printlnInfo( "All web sites have been created, waiting for downloads to finish" );
while (!executorService.awaitTermination( Long.MAX_VALUE, TimeUnit.MILLISECONDS )) {
// Nothing, just wait for termination again
}
printlnInfo( "Finished" );
}
private static Thread createDownloadsShutdownHook( ExecutorService executorService ) {
return new Thread( () -> {
if (executorService.isTerminated()) {
return; // Nothing to do
}
printlnInfo( "Stopping downloads" );
executorService.shutdownNow();
printlnInfo( "Waiting for running downloads to terminate" );
try {
while (!executorService.awaitTermination( Long.MAX_VALUE, TimeUnit.MILLISECONDS )) {
// Nothing, just wait for termination again
}
printlnInfo( "All running downloads have terminated" );
} catch (@SuppressWarnings("unused") InterruptedException e) {
Thread.currentThread().interrupt();
printlnWarning( "Some running downloads have been aborted, some temporary files may have been left behind" );
}
}, "Downloads shutdown hook" );
}
public static LocalDate getLatestComicStripDate() {
LocalDate latestUntestedDate = LocalDate.now( TIME_ZONE_COMIC_STRIP );
LocalDate endDateExcluding = latestUntestedDate.plusDays( 1 );
LocalDate result = null;
// Including today, test the last n days to get a quick result if possible
while ((ChronoUnit.DAYS.between( latestUntestedDate, endDateExcluding ) <= LATEST_COMIC_STRIP_QUICK_SCAN_LAST_DAYS)
&& (latestUntestedDate.compareTo( LATEST_COMIC_STRIP_DATE_KNOWN ) >= 0)) { // Only walk back to including latest-known-good
if (isComicStripOnline( latestUntestedDate )) {
printlnDebug( "Latest comic strip is from " + latestUntestedDate );
return latestUntestedDate;
}
// Comic strip for this day is not online yet: Try one day earlier
latestUntestedDate = latestUntestedDate.minusDays( 1 );
}
printlnInfo( "No online comic strips found for the last " + LATEST_COMIC_STRIP_QUICK_SCAN_LAST_DAYS + " days "
+ "(" + latestUntestedDate.plusDays( 1 ) + ".." + endDateExcluding.minusDays( 1 ) + ")" );
// Weird
// Rule out e.g. connection issues or changed comic strip URLs before continuing with a broader search
// Test a latest known good date
if ((latestUntestedDate.plusDays( 1 ).equals( LATEST_COMIC_STRIP_DATE_KNOWN )) // I.e. latest-known-good was found to be offline in the preceding loop already
|| !isComicStripOnline( LATEST_COMIC_STRIP_DATE_KNOWN )) {
throw new RuntimeException( "Unable to determine most recent comic strip date: "
+ "The known-to-exist comic strip for " + LATEST_COMIC_STRIP_DATE_KNOWN + " is unreachable, aborting" );
}
printlnInfo( "Sanity check: Known-to-exist comic strip for " + LATEST_COMIC_STRIP_DATE_KNOWN + " is online" );
// The comic strip appears to have been discontinued :(
// Do a binary search in the range day-after-latest-known-good..latestUntestedDate, including
if (ChronoUnit.DAYS.between( LATEST_COMIC_STRIP_DATE_KNOWN.plusDays( 1 ), latestUntestedDate.plusDays( 1 ) ) > 0) {
printlnInfo( "Binary searching for a newer latest comic strip in range "
+ "" + LATEST_COMIC_STRIP_DATE_KNOWN.plusDays( 1 ) + ".." + latestUntestedDate + "" );
result = getLatestComicStripDateBinarySearch( LATEST_COMIC_STRIP_DATE_KNOWN.plusDays( 1 ), latestUntestedDate.plusDays( 1 ) );
}
// The latest comic strip date is latest-known-good if the binary search did not yield anything newer
result = (result == null) ? LATEST_COMIC_STRIP_DATE_KNOWN : result;
printlnDebug( "Latest comic strip is from " + result );
return result;
}
private static LocalDate getLatestComicStripDateBinarySearch( LocalDate startDateIncluding, LocalDate endDateExcluding ) {
LocalDate startIncl = startDateIncluding;
LocalDate endExcl = endDateExcluding;
LocalDate result = null;
LocalDate testing;
while ((testing = getMiddle( startIncl, endExcl )) != null) {
if (isComicStripOnline( testing )) {
result = testing;
startIncl = testing.plusDays( 1 );
} else {
endExcl = testing;
}
}
return result;
}
private static LocalDate getMiddle( LocalDate startDateIncluding, LocalDate endDateExcluding ) {
if (startDateIncluding.compareTo( endDateExcluding ) >= 0) {
// Start date equal to or greater than end date
return null;
}
return startDateIncluding.plusDays( ChronoUnit.DAYS.between( startDateIncluding, endDateExcluding ) / 2 );
}
private static boolean isComicStripOnline( LocalDate forDay ) {
if (KNOWN_MISSING_COMIC_STRIPS.contains( forDay )) {
// Treat known missing comic strips as being "online", in the sense that they are
// where they are supposed to be – that is, non-existent
return true;
}
try (InputStream inputStream = new URL( FORMATTER_COMIC_STRIP_URL.format( forDay ) ).openStream()) {
printlnDebug( forDay + " is online" );
return true;
} catch (@SuppressWarnings("unused") IOException e) {
printlnDebug( forDay + " is offline" );
return false;
}
}
/**
* @param subdirName
* The name of the sub-directory into which to generate the month pages, also the name without suffix of the index .xhtml site
* @param minDaysIntoFuture
* <code>null</code> to generate a web site up to {@link #greatestDateIncl}, otherwise generate n days into the future including the complete
* month that future end day happens to lie in
* @param urlFactory
* A {@link Function} that calculates a comic strip image URL for a given {@link LocalDate}
*/
// TODO Maybe use XML processing instead of line-based text search-and-replace?
private void createWebSite( String subdirName, Integer minDaysIntoFuture, Function<LocalDate, String> urlFactory ) throws IOException {
LocalDate upToDayIncl = (minDaysIntoFuture != null)
? greatestDateIncl.plusDays( minDaysIntoFuture.intValue() ).with( TemporalAdjusters.lastDayOfMonth() )
: greatestDateIncl;
printlnInfo( "Creating web site for date range " + FIRST_COMIC_STRIP_DATE + ".." + upToDayIncl + ": \"" + getIndexFileName( subdirName ) + "\"" );
Files.createDirectories( targetDir.resolve( subdirName ) );
createStyles( subdirName );
Iterator<YearMonth> months = createStreamOfMonths( upToDayIncl ).iterator();
while (months.hasNext()) {
createMonthPage( subdirName, months.next(), upToDayIncl, AUTO_SKIP_FULL_MONTH_PAGES, urlFactory );
}
createIndexPage( subdirName, upToDayIncl );
}
private void createStyles( String subdirName ) throws IOException {
String stylesTemplateResourceName = "templates/styles.css";
Path stylesFile = getStylesFile( subdirName );
Path stylesFileTmp = stylesFile.getParent().resolve( stylesFile.getFileName() + TEMP_FILE_NAME_SUFFIX );
stylesFileTmp.toFile().deleteOnExit();
try (ReadableByteChannel inputChannel = Channels.newChannel( GetGarfield.class.getResourceAsStream( stylesTemplateResourceName ) );
SeekableByteChannel outputChannel = Files.newByteChannel( stylesFileTmp,
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE )) {
copy( inputChannel, outputChannel );
}
Files.move( stylesFileTmp, stylesFile, StandardCopyOption.REPLACE_EXISTING );
}
private Path getStylesFile( String subdirName ) {
return targetDir.resolve( subdirName ).resolve( STYLES_FILE_NAME );
}
private void createMonthPage( String subdirName, YearMonth month, LocalDate upToDayIncl, boolean autoSkip, Function<LocalDate, String> urlFactory )
throws IOException {
String monthTemplateResourceName = "templates/month.xhtml";
Path monthFile = getMonthFile( subdirName, month );
Path monthFileTmp = targetDir.resolve( monthFile.getFileName() + TEMP_FILE_NAME_SUFFIX );
Path monthFileFollwing = getMonthFile( subdirName, month.plusMonths( 1l ) );
boolean withAccessKeys = true;
if (autoSkip && existsAndIsNotDirectory( monthFile ) && existsAndIsNotDirectory( monthFileFollwing )) {
// If there is a file for both this month and the month afterwards, it can be safely assumed
// that this month's file contains the whole month, and that it therefore may be auto-skipped
return;
}
monthFileTmp.toFile().deleteOnExit();
try (ReadableByteChannel inputChannel = Channels.newChannel( GetGarfield.class.getResourceAsStream( monthTemplateResourceName ) );
BufferedReader reader = new BufferedReader( Channels.newReader( inputChannel, WEB_SITE_TEMPLATE_ENCODING_NAME ) );
SeekableByteChannel outputChannel = Files.newByteChannel( monthFileTmp,
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE );
BufferedWriter writer = new BufferedWriter( Channels.newWriter( outputChannel, WEB_SITE_TEMPLATE_ENCODING_NAME ) )) {
String line;
while ((line = reader.readLine()) != null) {
if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_TITLE )) {
writeMonthTitle( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_TITLE ) ), month );
} else if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_STYLES )) {
writeMonthStyles( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_STYLES ) ) );
} else if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_NAVIGATION )) {
writeMonthNavigation( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_NAVIGATION ) ),
("../" + getIndexFileName( subdirName )), month, upToDayIncl, withAccessKeys );
withAccessKeys = false;
} else if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_HEADING )) {
writeMonthHeading( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_HEADING ) ), month );
} else if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_COMICS )) {
writeMonthComicStrips( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_COMICS ) ), month, upToDayIncl, urlFactory );
} else {
writer.write( line );
writer.write( "\n" );
}
}
}
Files.move( monthFileTmp, monthFile, StandardCopyOption.REPLACE_EXISTING );
}
private void writeMonthTitle( BufferedWriter writer, String indentation, YearMonth forMonth ) throws IOException {
writer.write( indentation + "<title>Garfield " + FORMATTER_MONTH_PAGE_TITLE.format( forMonth ) + "</title>" + "\n" );
}
private void writeMonthStyles( BufferedWriter writer, String indentation ) throws IOException {
writer.write( indentation + "<link rel=\"stylesheet\" type=\"text/css\" href=\"" + STYLES_FILE_NAME + "\" />" + "\n" );
}
private void writeMonthNavigation( BufferedWriter writer, String indentation, String urlToIndex, YearMonth forMonth, LocalDate upToDayIncl,
boolean withAccessKeys )
throws IOException {
YearMonth previousMonth = forMonth.minusMonths( 1 );
YearMonth nextMonth = forMonth.plusMonths( 1 );
String textPreviousMonth = "← (a) " + FORMATTER_MONTH_PAGE_NAVIGATION.format( previousMonth );
String textIndex = "Index (w)";
String textNextMonth = FORMATTER_MONTH_PAGE_NAVIGATION.format( nextMonth ) + " (d) →";
String indentation1 = indentation + "\t";
writer.write( indentation + "<p>" );
{ // Visual aid to represent the generated text's indentation
writer.write( "\n" + indentation1 );
if (previousMonth.compareTo( YearMonth.from( FIRST_COMIC_STRIP_DATE ) ) >= 0) {
writer.write( "<a href=\"" + getMonthFileName( previousMonth ) + "\"" );
writer.write( withAccessKeys ? " accesskey=\"a\">" : ">" );
writer.write( textPreviousMonth + "</a>" );
} else {
writer.write( textPreviousMonth );
}
writer.write( "\n" + indentation1 );
writer.write( "<a href=\"" + urlToIndex + "\"" );
writer.write( withAccessKeys ? " accesskey=\"w\">" : ">" );
writer.write( textIndex + "</a>" );
writer.write( "\n" + indentation1 );
if (nextMonth.compareTo( YearMonth.from( upToDayIncl ) ) <= 0) {
writer.write( "<a href=\"" + getMonthFileName( nextMonth ) + "\"" );
writer.write( withAccessKeys ? " accesskey=\"d\">" : ">" );
writer.write( textNextMonth + "</a>" );
} else {
writer.write( textNextMonth );
}
}
writer.write( "\n" );
writer.write( indentation + "</p>" + "\n" );
}
private void writeMonthHeading( BufferedWriter writer, String indentation, YearMonth forMonth ) throws IOException {
writer.write( indentation + "<h1>" + FORMATTER_MONTH_PAGE_TITLE.format( forMonth ) + "</h1>" + "\n" );
}
private void writeMonthComicStrips( BufferedWriter writer, String indentation, YearMonth forMonth, LocalDate upToDayIncl,
Function<LocalDate, String> urlFactory ) throws IOException {
Iterator<LocalDate> days = createStreamOfDays( forMonth.atDay( 1 ), forMonth.plusMonths( 1 ).atDay( 1 ) ).iterator();
while (days.hasNext()) {
LocalDate day = days.next();
if (day.compareTo( FIRST_COMIC_STRIP_DATE ) < 0) {
continue;
}
if (day.compareTo( upToDayIncl ) > 0) {
break;
}
if (!KNOWN_MISSING_COMIC_STRIPS.contains( day )) {
writer.write( indentation + "<img" );
writer.write( " src=\"" + urlFactory.apply( day ) + "\"" );
writer.write( " id=\"" + FORMATTER_DAY_PAGE_COMIC_STRIP_ID.format( day ) + "\"" );
writer.write( " alt=\"" + FORMATTER_DAY_PAGE_COMIC_STRIP_ALT_TEXT.format( day ) + "\"" );
writer.write( " /><br />" + "\n" );
} else {
writer.write( indentation + "<span" );
writer.write( " id=\"" + FORMATTER_DAY_PAGE_COMIC_STRIP_ID.format( day ) + "\"" );
writer.write( ">" );
writer.write( FORMATTER_DAY_PAGE_COMIC_STRIP_MISSING.format( day ) );
writer.write( "</span><br />" + "\n" );
}
}
}
private Path getMonthFile( String subdirName, YearMonth month ) {
return targetDir.resolve( subdirName ).resolve( getMonthFileName( month ) );
}
private String getMonthFileName( YearMonth month ) {
return FORMATTER_MONTH_PAGE_FILE_NAME.format( month );
}
private void createIndexPage( String subdirName, LocalDate upToDayIncl ) throws IOException {
String indexTemplateResourceName = "templates/index.xhtml";
Path indexFile = getIndexFile( subdirName );
Path indexFileTmp = targetDir.resolve( indexFile.getFileName() + TEMP_FILE_NAME_SUFFIX );
indexFileTmp.toFile().deleteOnExit();
try (ReadableByteChannel inputChannel = Channels.newChannel( GetGarfield.class.getResourceAsStream( indexTemplateResourceName ) );
BufferedReader reader = new BufferedReader( Channels.newReader( inputChannel, WEB_SITE_TEMPLATE_ENCODING_NAME ) );
SeekableByteChannel outputChannel = Files.newByteChannel( indexFileTmp,
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE );
BufferedWriter writer = new BufferedWriter( Channels.newWriter( outputChannel, WEB_SITE_TEMPLATE_ENCODING_NAME ) )) {
String line;
while ((line = reader.readLine()) != null) {
if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_TITLE )) {
writeIndexTitle( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_TITLE ) ) );
} else if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_STYLES )) {
writeIndexStyles( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_STYLES ) ), subdirName );
} else if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_HEADING )) {
writeIndexHeading( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_HEADING ) ) );
} else if (line.endsWith( WEB_SITE_TEMPLATE_PLACEHOLDER_TOC )) {
writeIndexToc( writer, line.substring( 0, line.indexOf( WEB_SITE_TEMPLATE_PLACEHOLDER_TOC ) ), subdirName, upToDayIncl );
} else {
writer.write( line );
writer.write( "\n" );
}
}
}
Files.move( indexFileTmp, indexFile, StandardCopyOption.REPLACE_EXISTING );
}
private void writeIndexTitle( BufferedWriter writer, String indentation ) throws IOException {
writer.write( indentation + "<title>Garfield</title>" + "\n" );
}
private void writeIndexStyles( BufferedWriter writer, String indentation, String subdirName ) throws IOException {
writer.write( indentation + "<link rel=\"stylesheet\" type=\"text/css\" href=\"" + subdirName + "/" + STYLES_FILE_NAME + "\" />" + "\n" );
}
private void writeIndexHeading( BufferedWriter writer, String indentation ) throws IOException {
writer.write( indentation + "<h1>Index</h1>" + "\n" );
}
private void writeIndexToc( BufferedWriter writer, String indentation, String subdirName, LocalDate upToDayIncl ) throws IOException {
Iterator<Year> years = createStreamOfYears( upToDayIncl ).iterator();
String indentation1 = indentation + "\t";
while (years.hasNext()) {
Year year = years.next();
writer.write( indentation + "<h2>" + FORMATTER_YEAR_PAGE_TOC.format( year ) + "</h2>" + "\n" );
Iterator<YearMonth> monthsOfYear = createStreamOfMonths( year.atMonth( 1 ), year.plusYears( 1 ).atMonth( 1 ) ).iterator();
writer.write( indentation + "<p>" + "\n" );
while (monthsOfYear.hasNext()) {
YearMonth month = monthsOfYear.next();
if (month.compareTo( YearMonth.from( FIRST_COMIC_STRIP_DATE ) ) < 0) {
continue;
}
if (month.compareTo( YearMonth.from( upToDayIncl ) ) > 0) {
break;
}
writer.write( indentation1 );
writer.write( "<a href=\"" + subdirName + "/" + getMonthFileName( month ) + "\">" + FORMATTER_MONTH_PAGE_TOC.format( month ) + "</a>" );
writer.write( "\n" );
}
writer.write( indentation + "</p>" + "\n" );
}
}
private Path getIndexFile( String subdirName ) {
return targetDir.resolve( getIndexFileName( subdirName ) );
}
private static String getIndexFileName( String subdirName ) {
return subdirName + ".xhtml";
}
private void enqueueNextDownloadRunnable( ExecutorService executorService, Iterator<LocalDate> daysToDownload ) {
LocalDate dayToDownload = getNextSynchronized( daysToDownload );
if (dayToDownload == null) { // No more days to download a comic strip for
executorService.shutdown();
return;
}
try {
executorService.execute( createDownloadRunnable( executorService, daysToDownload, dayToDownload ) );
} catch (RejectedExecutionException e) {
// Do not complain if the ExecutorService has been shut down by a different download thread
if (!executorService.isShutdown()) {
throw e;
}
}
}
private Runnable createDownloadRunnable( ExecutorService executorService, Iterator<LocalDate> daysToDownload, LocalDate forDay ) {
return () -> {
enqueueNextDownloadRunnable( executorService, daysToDownload );
downloadComicStrip( forDay );
};
}
/**
* <p>
* Thread-safe method to get the next item from the given {@link Iterator}.
* </p>
*
* @param fromIterator
* The {@link Iterator} from which to get the next item
* @return The {@link Iterator#next() next item} or <code>null</code> if the {@link Iterator} {@link Iterator#hasNext() does not have any more items}
*/
private static <T> T getNextSynchronized( Iterator<T> fromIterator ) {
synchronized (fromIterator) {
return fromIterator.hasNext() ? fromIterator.next() : null;
}
}
private Stream<Year> createStreamOfYears( LocalDate upToDayIncl ) {
return createStreamOfYears( Year.from( FIRST_COMIC_STRIP_DATE ), Year.from( upToDayIncl ).plusYears( 1 ) );
}
private Stream<YearMonth> createStreamOfMonths( LocalDate upToDayIncl ) {
return createStreamOfMonths( YearMonth.from( FIRST_COMIC_STRIP_DATE ), YearMonth.from( upToDayIncl ).plusMonths( 1 ) );
}
private Stream<LocalDate> createStreamOfDays( LocalDate upToDayIncl ) {
return createStreamOfDays( FIRST_COMIC_STRIP_DATE, upToDayIncl.plusDays( 1 ) );
}
private static Stream<Year> createStreamOfYears( Year fromIncluding, Year toExcluding ) {
long yearsCount = ChronoUnit.YEARS.between( fromIncluding, toExcluding );
return Stream.iterate( fromIncluding, year -> year.plusYears( 1 ) ).limit( yearsCount );
}
private static Stream<YearMonth> createStreamOfMonths( YearMonth fromIncluding, YearMonth toExcluding ) {
long monthsCount = ChronoUnit.MONTHS.between( fromIncluding, toExcluding );
return Stream.iterate( fromIncluding, month -> month.plusMonths( 1 ) ).limit( monthsCount );
}
private static Stream<LocalDate> createStreamOfDays( LocalDate fromIncluding, LocalDate toExcluding ) {
long daysCount = ChronoUnit.DAYS.between( fromIncluding, toExcluding );
return Stream.iterate( fromIncluding, date -> date.plusDays( 1 ) ).limit( daysCount );
}
private void downloadComicStrip( LocalDate forDay ) {
Path targetFile = getComicStripFile( forDay );
Path targetFileTmp = targetFile.getParent().resolve( targetFile.getFileName() + TEMP_FILE_NAME_SUFFIX );
if (KNOWN_MISSING_COMIC_STRIPS.contains( forDay ) || Files.exists( targetFile )) {
return;
}
if (NO_DOWNLOAD) {
printlnWarning( "Not downloading " + targetFile.getFileName() + " (downloads disabled by constant)" );
try {
Thread.sleep( 200l + ((long) (160d * Math.random())) );
} catch (@SuppressWarnings("unused") InterruptedException e) {
Thread.currentThread().interrupt();
}
return;
}
try {
printlnInfo( "Downloading " + targetFile.getFileName() );
targetFileTmp.toFile().deleteOnExit();
download( new URL( FORMATTER_COMIC_STRIP_URL.format( forDay ) ), targetFileTmp );
Files.move( targetFileTmp, targetFile );
} catch (IOException e) {
printlnWarning( targetFile.getFileName() + ": Download failed (" + e.getClass().getSimpleName()
+ ((e.getLocalizedMessage() != null) ? (": " + e.getLocalizedMessage()) : "") + ")" );
}
}
private Path getComicStripFile( LocalDate forDay ) {
return subdirComicStrips.resolve( getComicStripFileName( forDay ) );
}
private static String getComicStripFileName( LocalDate forDay ) {
return FORMATTER_DAY_COMIC_STRIP_FILE_NAME.format( forDay );
}
private void download( URL url, Path toFile ) throws IOException {
try (ReadableByteChannel inputChannel = Channels.newChannel( url.openStream() );
SeekableByteChannel outputChannel = Files.newByteChannel( toFile,
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE )) {
copy( inputChannel, outputChannel );
}
}
private static void copy( ReadableByteChannel inputChannel, SeekableByteChannel outputChannel ) throws IOException {
ByteBuffer buffer = TL_BYTE_BUFFER.get();
while (inputChannel.read( clear( buffer ) ) >= 0) {
buffer.flip();
while (buffer.position() < buffer.limit()) {
outputChannel.write( buffer );
}
}
}
/**
* <p>
* {@link Buffer#clear() Clears} and returns the given {@link Buffer}.
* </p>
*
* @param buffer
* The {@link Buffer} that should be {@link Buffer#clear() cleared}
* @return The given {@link Buffer}
*/
private static <B extends Buffer> B clear( B buffer ) {
buffer.clear();
return buffer;
}
/**
* @return true if the given {@link Path} {@link Files#exists(Path, java.nio.file.LinkOption...) exists} and is not
* {@link Files#isDirectory(Path, LinkOption...) a directory}.
*/
private static boolean existsAndIsNotDirectory( Path path, LinkOption... linkOptions ) {
return Files.exists( path, linkOptions ) && !Files.isDirectory( path, linkOptions );
}
private static void printlnError( String message ) {
System.out.println( "ERROR " + message );
}
private static void printlnWarning( String message ) {
System.out.println( " WARN " + message );
}
private static void printlnInfo( String message ) {
System.out.println( " INFO " + message );
}
private static void printlnDebug( String message ) {
if (DEBUG) {
System.out.println( "DEBUG " + message );
}
}
}