import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

public class MeasureBuffer {

    /** Without a buffer: one system call per character. */
    static long withoutBuffer(String path) throws IOException {
        long n = 0;
        try (FileReader reader = new FileReader(path, StandardCharsets.UTF_8)) {
            while (reader.read() != -1) {
                n++;
            }
        }
        return n;
    }

    /** With a buffer, same loop: the buffer absorbs the calls. */
    static long withBuffer(String path) throws IOException {
        long n = 0;
        try (BufferedReader reader = new BufferedReader(
                new FileReader(path, StandardCharsets.UTF_8))) {
            while (reader.read() != -1) {
                n++;
            }
        }
        return n;
    }

    /** With a buffer and by lines: the idiomatic form. */
    static long byLines(String path) throws IOException {
        long n = 0;
        try (BufferedReader reader = new BufferedReader(
                new FileReader(path, StandardCharsets.UTF_8))) {
            String line;
            while ((line = reader.readLine()) != null) {
                n += line.length();
            }
        }
        return n;
    }

    public static void main(String[] args) throws IOException {
        String path = "data/catalog-large.txt";        // about 5 MB

        measure("Unbuffered, character by character", () -> withoutBuffer(path));
        measure("Buffered, character by character",   () -> withBuffer(path));
        measure("Buffered, by lines",                 () -> byLines(path));
    }

    interface Measurable { long run() throws IOException; }

    static void measure(String name, Measurable m) throws IOException {
        long start = System.nanoTime();
        long result = m.run();
        long ms = (System.nanoTime() - start) / 1_000_000;
        System.out.printf("%-35s %6d ms  (%d characters)%n", name, ms, result);
    }
}

Unbuffered, character by character     4512 ms  (5242880 characters)
Buffered, character by character         68 ms  (5242880 characters)
Buffered, by lines                       41 ms  (5158400 characters)

import java.io.*;
import java.nio.charset.StandardCharsets;

// Reading, default size (8192 characters)
BufferedReader reader = new BufferedReader(
        new FileReader("data/catalog.txt", StandardCharsets.UTF_8));

// Reading, explicit size
BufferedReader large = new BufferedReader(
        new FileReader("data/catalog.txt", StandardCharsets.UTF_8), 65536);

// Writing
BufferedWriter writer = new BufferedWriter(
        new FileWriter("data/report.txt", StandardCharsets.UTF_8));

// Over a bridge, when the source is not a file (07-03)
BufferedReader fromConsole = new BufferedReader(
        new InputStreamReader(System.in, StandardCharsets.UTF_8));

public String readLine() throws IOException

try (BufferedReader reader = new BufferedReader(
        new FileReader(path, StandardCharsets.UTF_8))) {

    String line;
    while ((line = reader.readLine()) != null) {
        process(line);
    }
}

// WRONG 1: it stops at the first empty line
while (!(line = reader.readLine()).isEmpty()) { }
// Also: NullPointerException on reaching the end, because readLine returns null

// WRONG 2: it reads every line TWICE and skips every other one
while (reader.readLine() != null) {
    process(reader.readLine());       // this is the NEXT line
}

// WRONG 3: ready() does not mean "there is data left". Section 7
while (reader.ready()) {
    process(reader.readLine());
}

int lineNumber = 0;
String line;
while ((line = reader.readLine()) != null) {
    lineNumber++;

    String clean = line.trim();
    if (clean.isEmpty() || clean.startsWith("#")) {
        continue;                        // blanks and comments: 02-04
    }
    process(clean, lineNumber);          // the number, so the error can be reported
}

public boolean ready() throws IOException

// WRONG: incorrect use of ready()
try (BufferedReader reader = new BufferedReader(new FileReader(path, UTF_8))) {
    while (reader.ready()) {              // <-- BUG
        System.out.println(reader.readLine());
    }
}

public void mark(int readAheadLimit) throws IOException
public void reset() throws IOException
public boolean markSupported()

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

public class HeaderDetection {

    /**
     * Detects whether the file has a header; if it does not, the first
     * line is data and must NOT be lost.
     */
    public static void process(String path) throws IOException {
        try (BufferedReader reader = new BufferedReader(
                new FileReader(path, StandardCharsets.UTF_8))) {

            reader.mark(8192);                     // a generous margin
            String first = reader.readLine();

            boolean hasHeader = first != null
                    && (first.startsWith("#") || first.toLowerCase().contains("title"));

            if (!hasHeader) {
                reader.reset();                    // put the line back into the stream
                System.out.println("No header: the first line is data");
            } else {
                System.out.println("Header detected: " + first);
            }

            String line;
            while ((line = reader.readLine()) != null) {
                System.out.println("  data: " + line);
            }
        }
    }
}

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

public class BufferedWriting {

    public static void write(String path) throws IOException {
        try (BufferedWriter writer = new BufferedWriter(
                new FileWriter(path, StandardCharsets.UTF_8))) {

            writer.write("# BiblioTech catalogue");
            writer.newLine();                          // the SYSTEM separator

            writer.write("BOOK;978-0000000001;Effective Java");
            writer.newLine();

            writer.write("BOOK;978-0000000002;Design Patterns");
            writer.newLine();
        }
        // close() -> flush() -> the data reaches the disk (07-02)
    }
}

try (PrintWriter output = new PrintWriter(          // 3. formatting
        new BufferedWriter(                          // 2. buffer
            new FileWriter(path,                     // 1. file + charset
                    StandardCharsets.UTF_8)))) {

    output.printf("%-25s %8.2f%n", "Effective Java", 3.75);
}

// Option A: PrintWriter. Convenient, but errors have to be asked for.
try (PrintWriter output = new PrintWriter(
        new BufferedWriter(new FileWriter(path, UTF_8)))) {
    output.printf("%-25s %8.2f%n", title, fine);
    if (output.checkError()) {
        throw new IOException("Failed to write " + path);
    }
}

// Option B: BufferedWriter. More verbose, but errors propagate on their own.
try (BufferedWriter output = new BufferedWriter(new FileWriter(path, UTF_8))) {
    output.write(String.format("%-25s %8.2f", title, fine));
    output.newLine();
}

import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;

// Reading: equivalent to new BufferedReader(new InputStreamReader(
//          new FileInputStream(...), UTF_8))
try (BufferedReader reader = Files.newBufferedReader(
        Path.of("data/catalog.txt"), StandardCharsets.UTF_8)) {

    String line;
    while ((line = reader.readLine()) != null) {
        process(line);
    }
}

// Writing, with explicit options
try (BufferedWriter writer = Files.newBufferedWriter(
        Path.of("data/report.txt"), StandardCharsets.UTF_8)) {
    writer.write("...");
    writer.newLine();
}

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

public class StreamProcessing {

    /**
     * Traverses a file of ANY size with constant memory.
     *
     * At any instant there is in memory: the buffer (8 KB), the current line
     * and the accumulators. It makes no difference if the file is 1 MB or 50 GB.
     */
    public static Summary analyse(String path) throws IOException {
        long lines = 0;
        long characters = 0;
        long blankLines = 0;
        int  maxLength = 0;
        String longestLine = "";

        try (BufferedReader reader = new BufferedReader(
                new FileReader(path, StandardCharsets.UTF_8))) {

            String line;
            while ((line = reader.readLine()) != null) {
                lines++;
                characters += line.length();

                if (line.isBlank()) {
                    blankLines++;
                }
                if (line.length() > maxLength) {
                    maxLength = line.length();
                    longestLine = line;         // ONE is kept, not all of them
                }
                // 'line' becomes available to the collector on the next pass
            }
        }
        return new Summary(lines, characters, blankLines, maxLength, longestLine);
    }

    public record Summary(long lines, long characters, long blankLines,
                          int maxLength, String longestLine) { }
}

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;

public class ConsoleWithBufferedReader {

    public static void main(String[] args) throws IOException {
        // System.in is an InputStream (bytes): the BRIDGE is needed (07-03)
        BufferedReader console = new BufferedReader(
                new InputStreamReader(System.in, StandardCharsets.UTF_8));

        System.out.print("Material reference: ");
        String reference = console.readLine();

        System.out.print("Loan days: ");
        String text = console.readLine();

        // BufferedReader does NOT parse: you have to convert by hand
        int days;
        try {
            days = Integer.parseInt(text.trim());
        } catch (NumberFormatException e) {
            System.out.println("'" + text + "' is not a valid number");
            return;
        }

        System.out.printf("Loan of %s for %d days%n", reference, days);

        // NOTE: the BufferedReader is NOT closed. Closing it would close System.in
        // for the whole application, with no way of reopening it (06-06).
    }
}

package com.nexussoftware.bibliotech.service;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.logging.Level;
import java.util.logging.Logger;

import com.nexussoftware.bibliotech.domain.Book;
import com.nexussoftware.bibliotech.domain.Material;
import com.nexussoftware.bibliotech.domain.DuplicateReferenceException;
import com.nexussoftware.bibliotech.infrastructure.FileFormat;

/**
 * Bulk import of the BiblioTech catalogue from a text file.
 *
 * Expected format (one line per material):
 *   BOOK;isbn;title;author;year
 *
 * Policy (06-07):
 *   - A bad line does NOT abort the import: it is discarded and counted.
 *   - A missing or unreadable file DOES abort: there is nothing to import.
 *   - Materials are accumulated and dumped into the catalogue AT THE END, so
 *     as not to leave it half done if the file turns out to be entirely corrupt.
 *
 * It processes in a STREAM: constant memory whatever the file size.
 */
public class CatalogImporter {

    private static final Logger LOG = Logger.getLogger(CatalogImporter.class.getName());

    private static final int EXPECTED_FIELDS = 5;
    private static final int MIN_YEAR = 1450;
    private static final int MAX_YEAR = 2100;

    /** If more than this fraction of lines fails, the file is suspicious. */
    private static final double CORRUPT_FILE_THRESHOLD = 0.5;

    /** Detailed errors that are kept; the rest are only counted. */
    private static final int MAX_DETAILED_ERRORS = 100;

    private final File file;

    public CatalogImporter(String path) {
        this.file = new File(Objects.requireNonNull(path, "The path cannot be null"));
    }

    // ------------------------- THE REPORT -------------------------

    /** Reason a line is discarded. An enum (04-07): no magic strings. */
    public enum DiscardReason {
        FIELD_COUNT("Incorrect number of fields"),
        UNKNOWN_TYPE("Material type not recognised"),
        EMPTY_FIELD("Mandatory field empty"),
        YEAR_NOT_NUMERIC("The year is not a number"),
        YEAR_OUT_OF_RANGE("Year outside the accepted range"),
        INVALID_ISBN("The ISBN does not have the expected format"),
        DUPLICATE("Reference or ISBN already present");

        private final String description;
        DiscardReason(String description) { this.description = description; }
        public String getDescription() { return description; }
    }

    /** A discarded line, with everything needed to correct it. */
    public record DiscardedLine(int number, DiscardReason reason, String detail,
                                String content) {
        @Override
        public String toString() {
            return String.format("Line %d [%s]: %s | %s",
                    number, reason.name(), detail, truncate(content));
        }
        private static String truncate(String s) {
            return (s.length() <= 60) ? s : s.substring(0, 57) + "...";
        }
    }

    /** Complete result of the import. */
    public static class Report {
        private final List<Material> imported = new ArrayList<>();
        private final List<DiscardedLine> discarded = new ArrayList<>();
        private final Map<DiscardReason, Integer> byReason = new LinkedHashMap<>();

        private int linesRead = 0;
        private int linesIgnored = 0;          // blanks and comments
        private int totalDiscards = 0;         // includes the undetailed ones
        private long milliseconds = 0;
        private boolean aborted = false;
        private String abortReason = null;

        void countLine()        { linesRead++; }
        void countIgnored()     { linesIgnored++; }
        void add(Material m)    { imported.add(m); }

        void discard(DiscardedLine d) {
            totalDiscards++;
            byReason.merge(d.reason(), 1, Integer::sum);        // 05-05
            if (discarded.size() < MAX_DETAILED_ERRORS) {
                discarded.add(d);
            }
        }

        void abort(String reason) {
            aborted = true;
            abortReason = reason;
        }

        public List<Material> getImported()             { return List.copyOf(imported); }
        public List<DiscardedLine> getDiscardedLines()  { return List.copyOf(discarded); }
        public Map<DiscardReason, Integer> getByReason() { return Map.copyOf(byReason); }
        public int getLinesRead()      { return linesRead; }
        public int getLinesIgnored()   { return linesIgnored; }
        public int getImportedTotal()  { return imported.size(); }
        public int getTotalDiscards()  { return totalDiscards; }
        public long getMilliseconds()  { return milliseconds; }
        public boolean isAborted()     { return aborted; }
        public String getAbortReason() { return abortReason; }

        public double successRate() {
            int processed = imported.size() + totalDiscards;
            return (processed == 0) ? 0.0 : (imported.size() * 100.0) / processed;
        }

        /** Readable report for the presentation layer. */
        public String summary() {
            StringBuilder sb = new StringBuilder();
            sb.append("=== IMPORT REPORT ===\n");

            if (aborted) {
                sb.append("  IMPORT ABORTED: ").append(abortReason).append('\n');
                return sb.toString();
            }

            sb.append(String.format("  Lines read       : %d%n", linesRead));
            sb.append(String.format("  Ignored          : %d (blank lines and comments)%n",
                    linesIgnored));
            sb.append(String.format("  Imported         : %d%n", imported.size()));
            sb.append(String.format("  Discarded        : %d%n", totalDiscards));
            sb.append(String.format("  Success rate     : %.1f%%%n", successRate()));
            sb.append(String.format("  Time             : %d ms%n", milliseconds));

            if (!byReason.isEmpty()) {
                sb.append("  --- Discards by reason ---\n");
                for (Map.Entry<DiscardReason, Integer> e : byReason.entrySet()) {
                    sb.append(String.format("    %-22s %4d  (%s)%n",
                            e.getKey().name(), e.getValue(),
                            e.getKey().getDescription()));
                }
            }
            if (!discarded.isEmpty()) {
                sb.append("  --- First discarded lines ---\n");
                int shown = Math.min(10, discarded.size());
                for (int i = 0; i < shown; i++) {
                    sb.append("    ").append(discarded.get(i)).append('\n');
                }
                if (totalDiscards > shown) {
                    sb.append(String.format("    ... and %d more%n",
                            totalDiscards - shown));
                }
            }
            return sb.toString();
        }
    }

    // ------------------------- THE IMPORT -------------------------

    /**
     * Imports the file into the given catalogue.
     *
     * @return the report. Never null, not even if it aborts.
     */
    public Report importInto(Catalog catalog) {
        Objects.requireNonNull(catalog, "The catalogue cannot be null");

        Report report = new Report();
        long start = System.nanoTime();

        LOG.info(() -> "Starting import from " + file.getAbsolutePath());

        // BufferedReader: constant memory even if the file has millions of
        // lines. Explicit charset, the same one the exporter uses.
        try (BufferedReader reader = new BufferedReader(
                new FileReader(file, FileFormat.CHARSET))) {

            String line;
            int number = 0;

            while ((line = reader.readLine()) != null) {    // the canonical loop
                number++;
                report.countLine();
                processLine(line, number, report);
            }

        } catch (java.io.FileNotFoundException e) {
            report.abort("The file " + file.getAbsolutePath() + " does not exist");
            LOG.log(Level.WARNING, "Import aborted: file not found", e);
            return finish(report, start);

        } catch (IOException e) {
            report.abort("I/O failure reading " + file.getAbsolutePath()
                    + ": " + e.getMessage());
            LOG.log(Level.SEVERE, "Import aborted by an I/O failure", e);
            return finish(report, start);
        }

        // Sanity check: if more than half fails, something is wrong with the
        // whole file (wrong charset, different format, columns swapped).
        // Better to reject it than to put rubbish into the catalogue.
        int processed = report.getImportedTotal() + report.getTotalDiscards();
        if (processed > 10
                && report.getTotalDiscards() > processed * CORRUPT_FILE_THRESHOLD) {

            report.abort(String.format(
                    "%.0f%% of the lines failed (%d of %d). The file does not seem "
                            + "to have the expected format; nothing is imported.",
                    100 - report.successRate(), report.getTotalDiscards(), processed));

            LOG.severe(() -> "Import rejected: " + report.getAbortReason());
            return finish(report, start);
        }

        // Dumping into the catalogue AT THE END: nothing has been touched until here
        flushToCatalog(catalog, report);
        return finish(report, start);
    }

    /** Validates and builds the material of a line. Never throws upwards. */
    private void processLine(String line, int number, Report report) {
        String clean = line.trim();

        if (clean.isEmpty() || clean.startsWith(FileFormat.COMMENT)) {
            report.countIgnored();
            return;
        }

        // The -1 keeps the trailing empty fields (07-01, solution 3)
        String[] fields = clean.split(FileFormat.FIELD_SEPARATOR, -1);

        if (fields.length != EXPECTED_FIELDS) {
            report.discard(new DiscardedLine(number, DiscardReason.FIELD_COUNT,
                    "there are " + fields.length + " and " + EXPECTED_FIELDS
                            + " were expected", clean));
            return;
        }

        String type   = fields[0].trim().toUpperCase();
        String isbn   = fields[1].trim();
        String title  = fields[2].trim();
        String author = fields[3].trim();
        String yearTx = fields[4].trim();

        if (!"BOOK".equals(type)) {
            report.discard(new DiscardedLine(number, DiscardReason.UNKNOWN_TYPE,
                    "'" + type + "'", clean));
            return;
        }
        if (title.isEmpty() || isbn.isEmpty()) {
            report.discard(new DiscardedLine(number, DiscardReason.EMPTY_FIELD,
                    title.isEmpty() ? "title" : "isbn", clean));
            return;
        }
        if (!isbn.matches("\\d{3}-\\d{10}")) {
            report.discard(new DiscardedLine(number, DiscardReason.INVALID_ISBN,
                    "'" + isbn + "' does not match NNN-NNNNNNNNNN", clean));
            return;
        }

        int year;
        try {
            year = Integer.parseInt(yearTx);
        } catch (NumberFormatException e) {
            report.discard(new DiscardedLine(number, DiscardReason.YEAR_NOT_NUMERIC,
                    "'" + yearTx + "'", clean));
            return;
        }
        if (year < MIN_YEAR || year > MAX_YEAR) {
            report.discard(new DiscardedLine(number, DiscardReason.YEAR_OUT_OF_RANGE,
                    year + " outside [" + MIN_YEAR + ", " + MAX_YEAR + "]", clean));
            return;
        }

        report.add(new Book(title, author.isEmpty() ? "Unknown" : author, isbn, year));
    }

    /**
     * Dumps what was imported into the catalogue.
     *
     * Duplicates are detected HERE, because only the catalogue knows about them.
     * A duplicate does not invalidate the whole import: it is discarded and we carry on.
     */
    private void flushToCatalog(Catalog catalog, Report report) {
        List<Material> pending = report.getImported();
        List<Material> accepted = new ArrayList<>(pending.size());

        for (Material m : pending) {
            try {
                catalog.register(m);
                accepted.add(m);
            } catch (DuplicateReferenceException e) {
                report.discard(new DiscardedLine(0, DiscardReason.DUPLICATE,
                        e.getMessage(), m.getReference()));
            }
        }
        report.imported.clear();
        report.imported.addAll(accepted);          // only what was really registered
    }

    private Report finish(Report report, long start) {
        report.milliseconds = (System.nanoTime() - start) / 1_000_000;

        if (report.isAborted()) {
            LOG.warning(() -> "Import aborted: " + report.getAbortReason());
        } else {
            LOG.info(() -> String.format(
                    "Import finished: %d imported, %d discarded, %d ms",
                    report.getImportedTotal(), report.getTotalDiscards(),
                    report.getMilliseconds()));
        }
        return report;
    }
}

package com.nexussoftware.bibliotech.presentation;

import com.nexussoftware.bibliotech.service.Catalog;
import com.nexussoftware.bibliotech.service.CatalogImporter;

public class ImportDemo {

    public static void main(String[] args) {
        Catalog catalog = new Catalog();

        CatalogImporter importer =
                new CatalogImporter("data/catalog-full.txt");

        CatalogImporter.Report report = importer.importInto(catalog);

        System.out.println(report.summary());
        System.out.println("Materials in the catalogue: " + catalog.size());
    }
}

=== IMPORT REPORT ===
  Lines read       : 5003
  Ignored          : 3 (blank lines and comments)
  Imported         : 4962
  Discarded        : 38
  Success rate     : 99.2%
  Time             : 87 ms
  --- Discards by reason ---
    YEAR_NOT_NUMERIC         14  (The year is not a number)
    INVALID_ISBN             11  (The ISBN does not have the expected format)
    FIELD_COUNT               7  (Incorrect number of fields)
    DUPLICATE                 4  (Reference or ISBN already present)
    EMPTY_FIELD               2  (Mandatory field empty)
  --- First discarded lines ---
    Line 47 [YEAR_NOT_NUMERIC]: 'nineteen hundred' | BOOK;978-0000000047;A book about something;Author Name;ni...
    Line 112 [FIELD_COUNT]: there are 4 and 5 were expected | BOOK;978-0000000112;Another;Author
    Line 340 [INVALID_ISBN]: '97800000340' does not match NNN-NNNNNNNNNN | BOOK;97800000340;A long enough title to be truncated here...
    ... and 35 more

package com.nexussoftware.bibliotech.util;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Statistics of a text file in A SINGLE PASS and with memory constant
 * with respect to the file size.
 *
 * The only thing that grows is the frequency map, which depends on the
 * number of DISTINCT words, not on the file size. A 10 GB file in English
 * has of the order of a hundred thousand distinct words: perfectly
 * manageable.
 */
public class FileStatistics {

    private static final int TOP_WORDS = 3;
    private static final int SAMPLE_LENGTH = 60;

    public record Statistics(
            long lines, long words, long characters,
            long blankLines, long commentLines,
            double averageLength, int minLength, int maxLength,
            int longestLineNumber, String longestLineSample,
            List<Map.Entry<String, Integer>> frequentWords) {

        public String report() {
            StringBuilder sb = new StringBuilder();
            sb.append("=== FILE STATISTICS ===\n");
            sb.append(String.format("  Lines             : %d%n", lines));
            sb.append(String.format("    blank           : %d%n", blankLines));
            sb.append(String.format("    comments        : %d%n", commentLines));
            sb.append(String.format("  Words             : %d%n", words));
            sb.append(String.format("  Characters        : %d%n", characters));
            sb.append(String.format("  Average length    : %.1f%n", averageLength));
            sb.append(String.format("  Minimum length    : %d%n", minLength));
            sb.append(String.format("  Maximum length    : %d (line %d)%n",
                    maxLength, longestLineNumber));
            sb.append(String.format("    sample          : %s%n", longestLineSample));
            sb.append("  Most frequent words:\n");
            for (Map.Entry<String, Integer> e : frequentWords) {
                sb.append(String.format("    %-20s %6d%n", e.getKey(), e.getValue()));
            }
            return sb.toString();
        }
    }

    public static Statistics analyse(String path) throws IOException {
        long lines = 0, words = 0, characters = 0;
        long blanks = 0, comments = 0;
        int  minimum = Integer.MAX_VALUE, maximum = 0;
        int  longestNumber = 0;
        String longestSample = "";

        // Only this grows, and with the number of DISTINCT words
        Map<String, Integer> frequencies = new HashMap<>();

        try (BufferedReader reader = new BufferedReader(
                new FileReader(path, StandardCharsets.UTF_8))) {

            String line;
            while ((line = reader.readLine()) != null) {      // canonical loop
                lines++;
                characters += line.length();

                int length = line.length();
                if (length < minimum) { minimum = length; }
                if (length > maximum) {
                    maximum = length;
                    longestNumber = (int) lines;
                    longestSample = length <= SAMPLE_LENGTH
                            ? line
                            : line.substring(0, SAMPLE_LENGTH) + "...";
                }

                String clean = line.trim();
                if (clean.isEmpty()) {
                    blanks++;
                    continue;
                }
                if (clean.startsWith("#")) {
                    comments++;
                    continue;
                }

                // Splits on anything that is not a letter or a digit
                for (String word : clean.toLowerCase().split("[^\\p{L}\\p{N}]+")) {
                    if (word.length() > 2) {              // ignores short articles
                        words++;
                        frequencies.merge(word, 1, Integer::sum);      // 05-05
                    }
                }
                // 'line' is discarded here: constant memory
            }
        }

        if (minimum == Integer.MAX_VALUE) {
            minimum = 0;                                  // empty file
        }

        return new Statistics(
                lines, words, characters, blanks, comments,
                lines == 0 ? 0.0 : (double) characters / lines,
                minimum, maximum, longestNumber, longestSample,
                mostFrequent(frequencies, TOP_WORDS));
    }

    /**
     * The n entries with the highest value.
     *
     * Without the Streams API (10-04): list + Comparator + sort, which is
     * exactly what 05-09 covered.
     */
    private static List<Map.Entry<String, Integer>> mostFrequent(
            Map<String, Integer> frequencies, int n) {

        List<Map.Entry<String, Integer>> entries = new ArrayList<>(frequencies.entrySet());

        // Descending by value; on a tie, alphabetical by key
        entries.sort(Comparator
                .comparing(Map.Entry<String, Integer>::getValue).reversed()
                .thenComparing(Map.Entry::getKey));

        return List.copyOf(entries.subList(0, Math.min(n, entries.size())));
    }

    public static void main(String[] args) throws IOException {
        String path = (args.length > 0) ? args[0] : "data/catalog.txt";
        System.out.println(analyse(path).report());
    }
}

=== FILE STATISTICS ===
  Lines             : 5003
    blank           : 1
    comments        : 2
  Words             : 19842
  Characters        : 248915
  Average length    : 49.8
  Minimum length    : 0
  Maximum length    : 87 (line 1204)
    sample          : BOOK;978-0000001204;Introduction to distributed system archi...
  Most frequent words:
    book                   5000
    programming             412
    systems                 287

package com.nexussoftware.bibliotech.service;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.logging.Logger;

/**
 * Filters a catalogue file writing another one with the lines meeting
 * a criterion, WITHOUT loading either of them into memory.
 *
 * It reads and writes in a stream simultaneously: at any instant there is
 * one line and two 8 KB buffers in memory.
 */
public class CatalogFilter {

    private static final Logger LOG = Logger.getLogger(CatalogFilter.class.getName());

    private static final String COMMENT = "#";

    /**
     * Filtering criterion. Your own functional interface (04-06): it receives
     * the line and its number, in case the criterion depends on the position.
     */
    @FunctionalInterface
    public interface LinePredicate {
        boolean matches(String line, int number);
    }

    public record FilterResult(int read, int written, int discarded,
                               int commentsKept, long milliseconds) {

        public String summary() {
            return String.format(
                    "Filtered: %d read, %d written, %d discarded, "
                            + "%d comments kept (%d ms)",
                    read, written, discarded, commentsKept, milliseconds);
        }
    }

    /**
     * Filters source into target ATOMICALLY (07-02).
     *
     * If something fails halfway, the previous target is left untouched.
     */
    public FilterResult filter(String source, String target, LinePredicate criterion)
            throws IOException {

        long start = System.nanoTime();

        File targetFile = new File(target);
        File temp = new File(targetFile.getAbsolutePath() + ".tmp");

        int read = 0, written = 0, discarded = 0, comments = 0;
        boolean completed = false;

        try {
            // Simultaneous reading and writing, both buffered.
            // Both resources are declared in the same try-with-resources:
            // they close in reverse order, the writer first (06-06).
            try (BufferedReader reader = new BufferedReader(
                         new FileReader(source, StandardCharsets.UTF_8));
                 BufferedWriter writer = new BufferedWriter(
                         new FileWriter(temp, StandardCharsets.UTF_8))) {

                String line;
                int number = 0;

                while ((line = reader.readLine()) != null) {
                    number++;
                    read++;

                    String clean = line.trim();

                    // Comments and headers: always kept
                    if (clean.isEmpty() || clean.startsWith(COMMENT)) {
                        writer.write(line);
                        writer.newLine();
                        comments++;
                        continue;
                    }

                    if (criterion.matches(line, number)) {
                        writer.write(line);
                        writer.newLine();            // readLine does NOT return the break
                        written++;
                    } else {
                        discarded++;
                    }
                }
                // BufferedWriter throws real IOExceptions: no checkError
            }

            // Atomic replacement
            if (targetFile.exists() && !targetFile.delete()) {
                throw new IOException("Could not replace " + target);
            }
            if (!temp.renameTo(targetFile)) {
                throw new IOException("Could not rename " + temp.getName());
            }
            completed = true;

        } finally {
            if (!completed && temp.exists() && !temp.delete()) {
                LOG.warning(() -> "Temporary file left undeleted: " + temp.getAbsolutePath());
            }
        }

        FilterResult result = new FilterResult(read, written, discarded,
                comments, (System.nanoTime() - start) / 1_000_000);

        LOG.info(result::summary);
        return result;
    }

    // ------------------------- CRITERIA -------------------------

    /** Publication year after the given one. Field 5 of the format. */
    public static LinePredicate publishedAfter(int year) {
        return (line, number) -> {
            String[] fields = line.split(";", -1);
            if (fields.length < 5) {
                return false;
            }
            try {
                return Integer.parseInt(fields[4].trim()) > year;
            } catch (NumberFormatException e) {
                return false;               // unreadable year: does not match
            }
        };
    }

    /** The author contains the given text, ignoring case. */
    public static LinePredicate authorContains(String text) {
        String wanted = text.toLowerCase();
        return (line, number) -> {
            String[] fields = line.split(";", -1);
            return fields.length >= 4 && fields[3].toLowerCase().contains(wanted);
        };
    }

    /** Combination of two criteria (lambda composition, 04-05). */
    public static LinePredicate and(LinePredicate a, LinePredicate b) {
        return (line, number) -> a.matches(line, number) && b.matches(line, number);
    }

    public static void main(String[] args) throws IOException {
        CatalogFilter filter = new CatalogFilter();

        System.out.println(filter.filter(
                "data/catalog.txt",
                "data/catalog-recent.txt",
                publishedAfter(2000)).summary());

        System.out.println(filter.filter(
                "data/catalog.txt",
                "data/catalog-bloch.txt",
                authorContains("bloch")).summary());

        System.out.println(filter.filter(
                "data/catalog.txt",
                "data/catalog-recent-fowler.txt",
                and(publishedAfter(1995), authorContains("fowler"))).summary());
    }
}

Filtered: 5003 read, 3418 written, 1582 discarded, 3 comments kept (61 ms)
Filtered: 5003 read, 12 written, 4988 discarded, 3 comments kept (48 ms)
Filtered: 5003 read, 7 written, 4993 discarded, 3 comments kept (52 ms)

package com.nexussoftware.bibliotech.service;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Compares two catalogue files and reports additions, removals and changes.
 *
 * WHY THIS ALGORITHM DOES NEED MEMORY:
 * To know whether a material from file B exists in A, you have to be able to
 * look it up. Since streams are SEQUENTIAL and do not allow going back
 * (07-03), the only way to consult A while traversing B is to have A
 * indexed in memory.
 *
 * WHICH ONE TO LOAD: the smaller one. The big one is traversed in a stream.
 *
 * IF NEITHER FITS: there are two classic ways out. Sort both files by key
 * and do a "merge" traversing them in parallel, which needs constant
 * memory; or load only the KEYS (the ISBNs) with a hash of each line,
 * which takes a tiny fraction. The second is the usual one.
 */
public class CatalogComparator {

    private static final String SEPARATOR = ";";
    private static final String COMMENT = "#";

    /** Data of a material exactly as it appears in the file. */
    private record Item(String isbn, String title, String author, int year) {

        /** Which fields differ from another item with the same ISBN. */
        List<String> differencesWith(Item other) {
            List<String> changes = new ArrayList<>();
            if (!title.equals(other.title)) {
                changes.add("title: '" + title + "' -> '" + other.title + "'");
            }
            if (!author.equals(other.author)) {
                changes.add("author: '" + author + "' -> '" + other.author + "'");
            }
            if (year != other.year) {
                changes.add("year: " + year + " -> " + other.year);
            }
            return changes;
        }
    }

    public record Differences(List<String> removals, List<String> additions,
                              List<String> changes, int identical) {

        public String report() {
            StringBuilder sb = new StringBuilder();
            sb.append("=== CATALOGUE COMPARISON ===\n");
            sb.append(String.format("  Identical      : %d%n", identical));
            sb.append(String.format("  Additions      : %d%n", additions.size()));
            sb.append(String.format("  Removals       : %d%n", removals.size()));
            sb.append(String.format("  Changes        : %d%n", changes.size()));

            printBlock(sb, "ADDITIONS (only in the new one)", additions);
            printBlock(sb, "REMOVALS (only in the old one)", removals);
            printBlock(sb, "CHANGES", changes);
            return sb.toString();
        }

        private void printBlock(StringBuilder sb, String title, List<String> lines) {
            if (lines.isEmpty()) {
                return;
            }
            sb.append("  --- ").append(title).append(" ---\n");
            int shown = Math.min(10, lines.size());
            for (int i = 0; i < shown; i++) {
                sb.append("    ").append(lines.get(i)).append('\n');
            }
            if (lines.size() > shown) {
                sb.append(String.format("    ... and %d more%n", lines.size() - shown));
            }
        }
    }

    /**
     * Compares two catalogues.
     *
     * @param oldPath the one loaded INTO MEMORY (the smaller one, ideally)
     * @param newPath the one traversed IN A STREAM
     */
    public Differences compare(String oldPath, String newPath) throws IOException {

        // PHASE 1: index the old one. A single pass.
        Map<String, Item> oldItems = index(oldPath);

        List<String> additions = new ArrayList<>();
        List<String> changes = new ArrayList<>();
        int identical = 0;

        // PHASE 2: traverse the new one IN A STREAM, consulting the index
        try (BufferedReader reader = new BufferedReader(
                new FileReader(newPath, StandardCharsets.UTF_8))) {

            String line;
            while ((line = reader.readLine()) != null) {
                Item fresh = parse(line);
                if (fresh == null) {
                    continue;                        // comment or bad line
                }

                // remove() rather than get(): what is left at the end are the REMOVALS.
                // It saves a second pass and an extra structure.
                Item old = oldItems.remove(fresh.isbn());

                if (old == null) {
                    additions.add(fresh.isbn() + " - " + fresh.title());
                    continue;
                }

                List<String> diff = old.differencesWith(fresh);
                if (diff.isEmpty()) {
                    identical++;
                } else {
                    changes.add(fresh.isbn() + " - " + String.join("; ", diff));
                }
            }
        }

        // PHASE 3: what is left over in the index are the removals
        List<String> removals = new ArrayList<>();
        for (Item i : oldItems.values()) {
            removals.add(i.isbn() + " - " + i.title());
        }

        removals.sort(null);                         // natural order (05-09)
        additions.sort(null);
        changes.sort(null);

        return new Differences(removals, additions, changes, identical);
    }

    /** Loads a catalogue indexed by ISBN. A single pass, buffered. */
    private Map<String, Item> index(String path) throws IOException {
        Map<String, Item> index = new HashMap<>();

        try (BufferedReader reader = new BufferedReader(
                new FileReader(path, StandardCharsets.UTF_8))) {

            String line;
            while ((line = reader.readLine()) != null) {
                Item i = parse(line);
                if (i != null) {
                    index.put(i.isbn(), i);
                }
            }
        }
        return index;
    }

    /** Returns null if the line is not a valid record. */
    private Item parse(String line) {
        String clean = line.trim();
        if (clean.isEmpty() || clean.startsWith(COMMENT)) {
            return null;
        }
        String[] c = clean.split(SEPARATOR, -1);
        if (c.length != 5) {
            return null;
        }
        try {
            return new Item(c[1].trim(), c[2].trim(), c[3].trim(),
                    Integer.parseInt(c[4].trim()));
        } catch (NumberFormatException e) {
            return null;
        }
    }

    public static void main(String[] args) throws IOException {
        Differences d = new CatalogComparator()
                .compare("data/catalog-2025.txt", "data/catalog-2026.txt");
        System.out.println(d.report());
    }
}

=== CATALOGUE COMPARISON ===
  Identical      : 4871
  Additions      : 92
  Removals       : 37
  Changes        : 14
  --- ADDITIONS (only in the new one) ---
    978-0000005001 - Hexagonal architecture
    978-0000005002 - Modern Java
    ... and 90 more
  --- REMOVALS (only in the old one) ---
    978-0000000341 - COBOL handbook
    ... and 36 more
  --- CHANGES ---
    978-0000000002 - title: 'Cafe Society' -> 'Café Society'
    978-0000000047 - year: 2018 -> 2019
    ... and 12 more