/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.beam.runners.dataflow.worker.windmill.work.refresh;

import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap.toImmutableMap;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import javax.annotation.concurrent.ThreadSafe;
import org.apache.beam.runners.dataflow.worker.DataflowExecutionStateSampler;
import org.apache.beam.runners.dataflow.worker.streaming.ComputationState;
import org.apache.beam.runners.dataflow.worker.streaming.RefreshableWork;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Asynchronously GetData requests to Streaming Engine for all sufficiently old active work and
 * invalidates stuck commits.
 *
 * <p>This informs Windmill that processing is ongoing and the work should not be retried. The age
 * threshold is determined by {@link #activeWorkRefreshPeriodMillis}
 */
@ThreadSafe
@Internal
public final class ActiveWorkRefresher {
  private static final Logger LOG = LoggerFactory.getLogger(ActiveWorkRefresher.class);
  private static final String FAN_OUT_REFRESH_WORK_EXECUTOR_NAME =
      "FanOutActiveWorkRefreshExecutor-%d";

  private final Supplier<Instant> clock;
  private final int activeWorkRefreshPeriodMillis;
  private final Supplier<Collection<ComputationState>> computations;
  private final DataflowExecutionStateSampler sampler;
  private final int stuckCommitDurationMillis;
  private final HeartbeatTracker heartbeatTracker;
  private final ScheduledExecutorService activeWorkRefreshExecutor;
  private final ExecutorService fanOutActiveWorkRefreshExecutor;

  public ActiveWorkRefresher(
      Supplier<Instant> clock,
      int activeWorkRefreshPeriodMillis,
      int stuckCommitDurationMillis,
      Supplier<Collection<ComputationState>> computations,
      DataflowExecutionStateSampler sampler,
      ScheduledExecutorService activeWorkRefreshExecutor,
      HeartbeatTracker heartbeatTracker) {
    this.clock = clock;
    this.activeWorkRefreshPeriodMillis = activeWorkRefreshPeriodMillis;
    this.stuckCommitDurationMillis = stuckCommitDurationMillis;
    this.computations = computations;
    this.sampler = sampler;
    this.activeWorkRefreshExecutor = activeWorkRefreshExecutor;
    this.heartbeatTracker = heartbeatTracker;
    this.fanOutActiveWorkRefreshExecutor =
        Executors.newCachedThreadPool(
            new ThreadFactoryBuilder().setNameFormat(FAN_OUT_REFRESH_WORK_EXECUTOR_NAME).build());
  }

  @SuppressWarnings("FutureReturnValueIgnored")
  public void start() {
    if (activeWorkRefreshPeriodMillis > 0) {
      activeWorkRefreshExecutor.scheduleWithFixedDelay(
          () -> {
            try {
              refreshActiveWork();
            } catch (RuntimeException e) {
              LOG.warn("Failed to refresh active work: ", e);
            }
          },
          activeWorkRefreshPeriodMillis,
          activeWorkRefreshPeriodMillis,
          TimeUnit.MILLISECONDS);
    }

    if (stuckCommitDurationMillis > 0) {
      int periodMillis = Math.max(stuckCommitDurationMillis / 10, 100);
      activeWorkRefreshExecutor.scheduleWithFixedDelay(
          this::invalidateStuckCommits, periodMillis, periodMillis, TimeUnit.MILLISECONDS);
    }
  }

  public void stop() {
    if (activeWorkRefreshPeriodMillis > 0 || stuckCommitDurationMillis > 0) {
      activeWorkRefreshExecutor.shutdown();
      try {
        activeWorkRefreshExecutor.awaitTermination(300, TimeUnit.SECONDS);
      } catch (InterruptedException e) {
        activeWorkRefreshExecutor.shutdownNow();
      }
    }
  }

  private void invalidateStuckCommits() {
    Instant stuckCommitDeadline = clock.get().minus(Duration.millis(stuckCommitDurationMillis));
    for (ComputationState computationState : computations.get()) {
      computationState.invalidateStuckCommits(stuckCommitDeadline);
    }
  }

  private void refreshActiveWork() {
    Instant refreshDeadline = clock.get().minus(Duration.millis(activeWorkRefreshPeriodMillis));
    Map<HeartbeatSender, Heartbeats> heartbeatsBySender =
        aggregateHeartbeatsBySender(refreshDeadline);
    if (heartbeatsBySender.isEmpty()) {
      return;
    }

    List<CompletableFuture<Void>> fanOutRefreshActiveWork = new ArrayList<>();

    // Send the first heartbeat on the calling thread, and fan out the rest via the
    // fanOutActiveWorkRefreshExecutor.
    Map.@MonotonicNonNull Entry<HeartbeatSender, Heartbeats> firstHeartbeat = null;
    for (Map.Entry<HeartbeatSender, Heartbeats> heartbeat : heartbeatsBySender.entrySet()) {
      if (firstHeartbeat == null) {
        firstHeartbeat = heartbeat;
      } else {
        fanOutRefreshActiveWork.add(
            CompletableFuture.runAsync(
                () -> sendHeartbeatSafely(heartbeat), fanOutActiveWorkRefreshExecutor));
      }
    }

    sendHeartbeatSafely(checkNotNull(firstHeartbeat));
    fanOutRefreshActiveWork.forEach(CompletableFuture::join);
  }

  /** Aggregate the heartbeats across computations by HeartbeatSender for correct fan out. */
  private Map<HeartbeatSender, Heartbeats> aggregateHeartbeatsBySender(Instant refreshDeadline) {
    Map<HeartbeatSender, Heartbeats.Builder> heartbeatsBySender = new HashMap<>();

    // Aggregate the heartbeats across computations by HeartbeatSender for correct fan out.
    for (ComputationState computationState : computations.get()) {
      for (RefreshableWork work : computationState.getRefreshableWork(refreshDeadline)) {
        heartbeatsBySender
            .computeIfAbsent(work.heartbeatSender(), ignored -> Heartbeats.builder())
            .add(computationState.getComputationId(), work, sampler);
      }
    }

    return heartbeatsBySender.entrySet().stream()
        .collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue().build()));
  }

  /**
   * Send the {@link Heartbeats} using the {@link HeartbeatSender}. Safe since exceptions are caught
   * and logged.
   */
  private void sendHeartbeatSafely(Map.Entry<HeartbeatSender, Heartbeats> heartbeat) {
    try (AutoCloseable ignored = heartbeatTracker.trackHeartbeats(heartbeat.getValue().size())) {
      HeartbeatSender sender = heartbeat.getKey();
      Heartbeats heartbeats = heartbeat.getValue();
      sender.sendHeartbeats(heartbeats);
    } catch (Exception e) {
      LOG.error(
          "Unable to send {} heartbeats to {}.",
          heartbeat.getValue().size(),
          heartbeat.getKey(),
          e);
    }
  }

  @FunctionalInterface
  public interface HeartbeatTracker {
    AutoCloseable trackHeartbeats(int numHeartbeats);
  }
}
