/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.beam.examples.complete.game;

import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;
import org.apache.beam.examples.complete.game.utils.GameConstants;
import org.apache.beam.examples.complete.game.utils.WriteToText;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Filter;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.WithTimestamps;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

/**
 * This class is the second in a series of four pipelines that tell a story in a 'gaming' domain,
 * following {@link UserScore}. In addition to the concepts introduced in {@link UserScore}, new
 * concepts include: windowing and element timestamps; use of {@code Filter.by()}.
 *
 * <p>This pipeline processes data collected from gaming events in batch, building on {@link
 * UserScore} but using fixed windows. It calculates the sum of scores per team, for each window,
 * optionally allowing specification of two timestamps before and after which data is filtered out.
 * This allows a model where late data collected after the intended analysis window can be included,
 * and any late-arriving data prior to the beginning of the analysis window can be removed as well.
 * By using windowing and adding element timestamps, we can do finer-grained analysis than with the
 * {@link UserScore} pipeline. However, our batch processing is high-latency, in that we don't get
 * results from plays at the beginning of the batch's time period until the batch is processed.
 *
 * <p>To execute this pipeline, specify the pipeline configuration like this:
 *
 * <pre>{@code
 * --tempLocation=YOUR_TEMP_DIRECTORY
 * --runner=YOUR_RUNNER
 * --output=YOUR_OUTPUT_DIRECTORY
 * (possibly options specific to your runner or permissions for your temp/output locations)
 * }</pre>
 *
 * <p>Optionally include {@code --input} to specify the batch input file path. To indicate a time
 * after which the data should be filtered out, include the {@code --stopMin} arg. E.g., {@code
 * --stopMin=2015-10-18-23-59} indicates that any data timestamped after 23:59 PST on 2015-10-18
 * should not be included in the analysis. To indicate a time before which data should be filtered
 * out, include the {@code --startMin} arg. If you're using the default input specified in {@link
 * UserScore}, "gs://apache-beam-samples/game/gaming_data*.csv", then {@code
 * --startMin=2015-11-16-16-10 --stopMin=2015-11-17-16-10} are good values.
 */
public class HourlyTeamScore extends UserScore {

  private static DateTimeFormatter minFmt =
      DateTimeFormat.forPattern("yyyy-MM-dd-HH-mm")
          .withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/Los_Angeles")));

  /** Options supported by {@link HourlyTeamScore}. */
  public interface Options extends UserScore.Options {

    @Description("Numeric value of fixed window duration, in minutes")
    @Default.Integer(60)
    Integer getWindowDuration();

    void setWindowDuration(Integer value);

    @Description(
        "String representation of the first minute after which to generate results,"
            + "in the format: yyyy-MM-dd-HH-mm . This time should be in PST."
            + "Any input data timestamped prior to that minute won't be included in the sums.")
    @Default.String("1970-01-01-00-00")
    String getStartMin();

    void setStartMin(String value);

    @Description(
        "String representation of the first minute for which to not generate results,"
            + "in the format: yyyy-MM-dd-HH-mm . This time should be in PST."
            + "Any input data timestamped after that minute won't be included in the sums.")
    @Default.String("2100-01-01-00-00")
    String getStopMin();

    void setStopMin(String value);
  }

  /**
   * Create a map of information that describes how to write pipeline output to text. This map is
   * passed to the {@link WriteToText} constructor to write team score sums and includes information
   * about window start time.
   */
  protected static Map<String, WriteToText.FieldFn<KV<String, Integer>>> configureOutput() {
    Map<String, WriteToText.FieldFn<KV<String, Integer>>> config = new HashMap<>();
    config.put("team", (e, w, t, p) -> e.getKey());
    config.put("total_score", (e, w, t, p) -> e.getValue());
    config.put(
        "window_start",
        (e, w, t, p) -> {
          IntervalWindow window = (IntervalWindow) w;
          return GameConstants.DATE_TIME_FORMATTER.print(window.start());
        });
    return config;
  }

  /** Run a batch pipeline to do windowed analysis of the data. */
  // [START DocInclude_HTSMain]
  public static void main(String[] args) throws Exception {
    // Begin constructing a pipeline configured by commandline flags.
    Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
    Pipeline pipeline = Pipeline.create(options);

    final Instant stopMinTimestamp = new Instant(minFmt.parseMillis(options.getStopMin()));
    final Instant startMinTimestamp = new Instant(minFmt.parseMillis(options.getStartMin()));

    // Read 'gaming' events from a text file.
    pipeline
        .apply(TextIO.read().from(options.getInput()))
        // Parse the incoming data.
        .apply("ParseGameEvent", ParDo.of(new ParseEventFn()))

        // Filter out data before and after the given times so that it is not included
        // in the calculations. As we collect data in batches (say, by day), the batch for the day
        // that we want to analyze could potentially include some late-arriving data from the
        // previous day.
        // If so, we want to weed it out. Similarly, if we include data from the following day
        // (to scoop up late-arriving events from the day we're analyzing), we need to weed out
        // events that fall after the time period we want to analyze.
        // [START DocInclude_HTSFilters]
        .apply(
            "FilterStartTime",
            Filter.by(
                (GameActionInfo gInfo) -> gInfo.getTimestamp() > startMinTimestamp.getMillis()))
        .apply(
            "FilterEndTime",
            Filter.by(
                (GameActionInfo gInfo) -> gInfo.getTimestamp() < stopMinTimestamp.getMillis()))
        // [END DocInclude_HTSFilters]

        // [START DocInclude_HTSAddTsAndWindow]
        // Add an element timestamp based on the event log, and apply fixed windowing.
        .apply(
            "AddEventTimestamps",
            WithTimestamps.of((GameActionInfo i) -> new Instant(i.getTimestamp())))
        .apply(
            "FixedWindowsTeam",
            Window.into(FixedWindows.of(Duration.standardMinutes(options.getWindowDuration()))))
        // [END DocInclude_HTSAddTsAndWindow]

        // Extract and sum teamname/score pairs from the event data.
        .apply("ExtractTeamScore", new ExtractAndSumScore("team"))
        .apply(
            "WriteTeamScoreSums", new WriteToText<>(options.getOutput(), configureOutput(), true));

    pipeline.run().waitUntilFinish();
  }
  // [END DocInclude_HTSMain]

}
