% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/replace_mnirs.R
\name{replace_mnirs}
\alias{replace_mnirs}
\alias{replace_invalid}
\alias{replace_outliers}
\alias{replace_missing}
\title{Replace outliers, invalid, and missing values in \emph{{mnirs}} data}
\usage{
replace_mnirs(
  data,
  nirs_channels = NULL,
  time_channel = NULL,
  invalid_values = NULL,
  invalid_above = NULL,
  invalid_below = NULL,
  outlier_cutoff = NULL,
  width = NULL,
  span = NULL,
  method = c("linear", "median", "locf", "none"),
  verbose = TRUE
)

replace_invalid(
  x,
  t = seq_along(x),
  invalid_values = NULL,
  invalid_above = NULL,
  invalid_below = NULL,
  width = NULL,
  span = NULL,
  method = c("median", "none"),
  verbose = TRUE,
  ...
)

replace_outliers(
  x,
  t = seq_along(x),
  outlier_cutoff = 3,
  width = NULL,
  span = NULL,
  method = c("median", "none"),
  verbose = TRUE,
  ...
)

replace_missing(
  x,
  t = seq_along(x),
  width = NULL,
  span = NULL,
  method = c("linear", "median", "locf"),
  verbose = TRUE,
  ...
)
}
\arguments{
\item{data}{A data frame of class \emph{"mnirs"} containing time series data and
metadata.}

\item{nirs_channels}{A character vector giving the names of mNIRS columns to
operate on. Must match column names in \code{data} exactly.
\itemize{
\item If \code{NULL} (default), the \code{nirs_channels} metadata attribute of \code{data} is
used.
}}

\item{time_channel}{A character string naming the time or sample column.
Must match a column name in \code{data} exactly.
\itemize{
\item If \code{NULL} (default), the \code{time_channel} metadata attribute of \code{data} is
used.
}}

\item{invalid_values}{A numeric vector of invalid values to be replaced,
e.g. \code{invalid_values = c(0, 100, 102.3)}. Default \code{NULL} will not replace
invalid values.}

\item{invalid_above, invalid_below}{Numeric values each specifying cutoff
values, above or below which (respectively) will be replaced, \emph{inclusive}
of the specified cutoff values.}

\item{outlier_cutoff}{A numeric value for the local outlier threshold, as
the number of standard deviations from the local median.
\itemize{
\item Default \code{NULL} will not replace outliers.
\item Lower values are more sensitive and flag more outliers; higher values
are more conservative.
\item \code{outlier_cutoff = 3} Pearson's 3 sigma edit rule.
\code{outlier_cutoff = 2} approximates a Tukey-style 1.5*IQR rule.
\code{outlier_cutoff = 0} Tukey's median filter.
}}

\item{width}{An integer defining the local window in number of samples
centred on \code{idx}, between
\verb{[idx - floor(width/2), idx + floor(width/2)]}.}

\item{span}{A numeric value defining the local window time span around
\code{idx} in units of \code{time_channel} or \code{t}, between
\verb{[t - span/2, t + span/2]}.}

\item{method}{A character string indicating how to handle \code{NA}
replacement (see \emph{Details}):
\describe{
\item{\code{"linear"}}{Replaces \code{NA}s via linear interpolation (the
\emph{default}) using \code{\link[stats:approxfun]{stats::approx()}}.}
\item{\code{"median"}}{Replaces \code{NA}s with the local median of valid
values within a centred window defined by \code{width} or \code{span}.}
\item{\code{"locf"}}{\emph{"Last observation carried forward"}. Replaces
\code{NA}s with the most recent valid value to the left for trailing
\code{NA}s or to the right for leading \code{NA}s, using
\code{\link[stats:approxfun]{stats::approx()}}.}
\item{\code{"none"}}{Returns \code{NA}s without replacement.}
}}

\item{verbose}{Logical. Default is \code{TRUE}. Display or silence (if \code{FALSE})
warnings and information messages helpful for troubleshooting. Ad
global default can be set via \code{options(mnirs.verbose = FALSE)}.}

\item{x}{A numeric vector of the response variable.}

\item{t}{An \emph{optional} numeric vector of the predictor variable (e.g. time).
Default is \code{seq_along(x)}.}

\item{...}{Additional arguments.}
}
\value{
\code{replace_mnirs()} return a \link[tibble:tibble-package]{tibble} of
class \code{"mnirs"} with metadata available via \code{attributes()}.

\code{replace_invalid()} returns a numeric vector the same length as
\code{x} with invalid values replaced.

\code{replace_outliers()} returns a numeric vector the same length as
\code{x} with outliers replaced.

\code{replace_missing()} returns a numeric vector the same length as
\code{x} with missing values replaced.
}
\description{
Detect and replace local outliers, specified invalid values, and missing
\code{NA} values across \code{nirs_channels} within an \emph{"mnirs"} data frame.
\code{replace_mnirs()} operates on a data frame, extending the vectorised
functions:.

\code{replace_invalid()} detects specified invalid values or range cutoffs in a
numeric vector and replace them with the local median value or \code{NA}.

\code{replace_outliers()} detects local outliers in a numeric vector using a Hampel filter and replaces with the local median value or \code{NA}.

\code{replace_missing()} detects missing (\code{NA}) values in a numeric vector and
replaces via interpolation.
}
\details{
\subsection{Automatic channel detection}{

\code{nirs_channels} and \code{time_channel} are retrieved automatically from
\emph{"mnirs"} metadata if not specified explicitly. Columns in \code{data} not
listed in \code{nirs_channels} are passed through unprocessed.
}

\subsection{The rolling window}{

\code{replace_outliers()} and \code{replace_missing()} (when \code{method = "median"})
operate over a local rolling window for outlier detection and median
interpolation. The window is specified by either \code{width} as the number
of samples, or \code{span} as the time span in units of \code{time_channel}.
A partial window is calculated at the edges of the data.
}

\subsection{Replace invalid values with with replace_invalid()}{

Specific \code{invalid_values} can be replaced, such as \code{c(0, 100, 102.3)}.
Data ranges can be replaced with cutoff values specified by \code{invalid_above}
and \code{invalid_below}, where any values higher or lower than the specified
cutoff values (respectively) will be replaced, \emph{inclusive} of the cutoff
values themselves.
}

\subsection{Outlier detection with replace_outliers()}{

Rolling local medians are computed across \code{x} within a window defined
by \code{width} (number of samples) or \code{span} (time span in units of \code{t}).

Outliers are detected with robust median absolute deviation (MAD),
adapted from \code{pracma::hampel()}. Deviations equal to or less than the
smallest absolute time series difference in \code{x} are excluded, to avoid
flagging negligible differences where local data have minimal or zero
variation.
}

\subsection{Replacement behaviour}{

Values of \code{x} outside the local bounds defined by \code{outlier_cutoff} are
identified as outliers and either replaced with the local median
(\code{method = "median"}, the \emph{default}) or set to \code{NA} (\code{method = "none"}).

Existing \code{NA} values in \code{x} are \emph{not} replaced. They are passed
through to the returned vector. See \code{\link[=replace_missing]{replace_missing()}}.
}

\subsection{Choosing outlier_cutoff}{

\code{outlier_cutoff} is the number of (MAD-normalised) standard deviations
from the local median. Higher values are more conservative; lower
values flag more outliers.
\itemize{
\item \code{outlier_cutoff = 3} -- Pearson's 3 sigma edit rule (default).
\item \code{outlier_cutoff = 2} -- approximately Tukey-style 1.5*IQR rule.
\item \code{outlier_cutoff = 0} -- Tukey's median filter (every point
replaced by local median).
}
}

\subsection{Interpolation with replace_missing()}{

\code{method = "linear"} and \code{method = "locf"} use \code{\link[stats:approxfun]{stats::approx()}} with
\code{rule = 2}, so leading \code{NA}s are filled by \emph{"nocb"}
(\emph{"next observation carried backward"}) and trailing \code{NA}s by \emph{"locf"}.

\code{method = "median"} calculates the local median of valid (non-\code{NA})
values to either side of \code{NA}s, within a window defined by \code{width}
(number of samples) or \code{span} (time span in units of \code{t}). Sequential
\code{NA}s are all replaced by the same median value.
}

\subsection{Edge behaviour for method = "median"}{

If there are no valid values within \code{span} to one side of the \code{NA},
the median of the other side is used (i.e. for leading and trailing
\code{NA}s). If there are no valid values within either side, the first
valid sample on either side is used (equivalent to
\code{replace_missing(x, width = 1)}).
}
}
\examples{
## vectorised operations
x <- c(1, 999, 3, 4, 999, 6)
replace_invalid(x, invalid_values = 999, width = 3, method = "median")

(x_na <- replace_outliers(x, outlier_cutoff = 3, width = 3, method = "none"))

replace_missing(x_na, method = "linear")

## read example data
data <- read_mnirs(
    file_path = example_mnirs("moxy_ramp"),
    nirs_channels = c(smo2 = "SmO2 Live"),
    time_channel = c(time = "hh:mm:ss"),
    verbose = FALSE
)

## clean data
data_clean <- replace_mnirs(
    data,                  ## channels retrieved from metadata
    invalid_values = 0,    ## known invalid values in the data
    invalid_above = 90,    ## remove data spikes above 90
    outlier_cutoff = 3,    ## Pearson's 3 sigma edit rule
    width = 7,            ## window for outlier detection and interpolation
    method = "linear"      ## linear interpolation over NAs
)

\donttest{
  if (requireNamespace("ggplot2", quietly = TRUE)) {
    ## plot original and show where values have been replaced
    ## ignore warning about replacing the existing colour scale
    plot(data, time_labels = TRUE) +
      ggplot2::scale_colour_manual(
        name = NULL,
        breaks = c("smo2", "replaced"),
        values = palette_mnirs(2)
      ) +
      ggplot2::geom_point(
        data = data[data_clean$smo2 != data$smo2, ],
        ggplot2::aes(y = smo2, colour = "replaced"),
        na.rm = TRUE
      ) +
      ggplot2::geom_line(
        data = {
          data_clean[!is.na(data$smo2), "smo2"] <- NA
          data_clean
        },
        ggplot2::aes(y = smo2, colour = "replaced"),
        linewidth = 1, na.rm = TRUE
      )
  }
}

}
