% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/learn.R
\name{learn}
\alias{learn}
\title{Monte Carlo computation of posterior probability distribution}
\usage{
learn(
  data,
  metadata,
  auxdata = NULL,
  outputdir = NULL,
  nsamples = 3600,
  nchains = 8,
  nsamplesperchain = 450,
  parallel = TRUE,
  seed = NULL,
  cleanup = TRUE,
  appendinfo = TRUE,
  valueislearnt = TRUE,
  subsampledata = NULL,
  prior = missing(data) || is.null(data),
  startupMCiterations = 3600,
  minMCiterations = 0,
  maxMCiterations = +Inf,
  maxhours = +Inf,
  ncheckpoints = 12,
  maxrelMCSE = +Inf,
  minESS = 450,
  initES = 2,
  thinning = NULL,
  verbose = TRUE,
  plottraces = !cleanup,
  showKtraces = FALSE,
  showAlphatraces = FALSE,
  hyperparams = list(ncomponents = 64, minalpha = -4, maxalpha = 4, byalpha = 1, Rshapelo
    = 0.5, Rshapehi = 0.5, Rvarm1 = 3^2, Cshapelo = 0.5, Cshapehi = 0.5, Cvarm1 = 3^2,
    Dshapelo = 0.5, Dshapehi = 0.5, Dvarm1 = 3^2, Bshapelo = 1, Bshapehi = 1, Dthreshold
    = 1, tscalefactor = 4.266, Oprior = "Hadamard", Nprior = "Hadamard", avoidzeroW =
    NULL, initmethod = "datacentre", Qerror = pnorm(c(-1, 1)))
)
}
\arguments{
\item{data}{A dataset, given as a \code{\link[base:data.frame]{base::data.frame()}} or as a file path to a CSV file. If missing or \code{NULL}, then the prior probability distribution is calculated.}

\item{metadata}{\link{metadata} about the dataset's variates, given either as a \link[base:data.frame]{data frame} or as a file path to a CSV file.}

\item{auxdata}{A larger dataset, given as a data frame or as a file path to a CSV file. Such a dataset would be too large to use in the Monte Carlo sampling, but can still be used to help estimate some hyperparameters.}

\item{outputdir}{\code{NULL} (default) or \code{NA} or character: path to folder where output information and diagnostics should be saved. If \code{NULL}, a directory is created in the temporary-directory space given by \code{\link[base:tempdir]{base::tempdir()}}. If \code{NA}, a directory is created in the current working directory given by \code{\link[base:getwd]{base::getwd()}}. If character, this is taken to be the output directory; it should of course be writable by the user.}

\item{nsamples}{Integer, default 3600: number of desired, \emph{approximately independent} Monte Carlo samples. If this argument is changed, the user is also required to explicitly give either \code{nchains} or \code{nsamplesperchain}, but not both; the remaining third argument is determined from \eqn{\mathrm{nsamples} = \mathrm{nchains} \times \mathrm{nsamplesperchain}}.}

\item{nchains}{Integer, default 8: number of Monte Carlo chains. If this argument is changed, the user is also required to explicitly give either \code{nsamples} or \code{nsamplesperchain}, but not both; the remaining third argument is determined from \eqn{\mathrm{nsamples} = \mathrm{nchains} \times \mathrm{nsamplesperchain}}.}

\item{nsamplesperchain}{Integer, default 450: number of \emph{approximately independent} Monte Carlo samples per chain. If this argument is changed, the user is also required to explicitly give either \code{nsamples} or \code{nchains}, but not both; the remaining third argument is determined from \eqn{\mathrm{nsamples} = \mathrm{nchains} \times \mathrm{nsamplesperchain}}.}

\item{parallel}{Logical or positive integer or cluster object. \code{TRUE} (default): use roughly half of available cores; \code{FALSE} (default): use serial computation; integer: use this many cores. It can also be a cluster object previously created with \code{\link[parallel:makeCluster]{parallel::makeCluster()}}; in this case the parallel computation will use this object.}

\item{seed}{Integer or \code{NULL} (default): use this seed for the random number generator. If \code{NULL}, do not set the seed.}

\item{cleanup}{Logical, default \code{TRUE}: remove diagnostic files at the end of the computation?}

\item{appendinfo}{Logical, default \code{TRUE}: append information about number of variates ('V'), number of data points ('D'), number of Monte Carlo samples ('S'), and timestamp, to the name of the output directory \code{outputdir}? The appended string has the format 'Vn_Dn_Sn_YYMMDDTHHMMSS'.}

\item{valueislearnt}{Logical or \code{NULL}: should the \code{VALUE} returned be the \code{learnt} object containing the results from the Monte Carlo computation? Default \code{TRUE}. If \code{FALSE}, then \code{VALUE} is the output directory name. If \code{NULL}, then \code{VALUE} is \code{NULL}.}

\item{subsampledata}{Integer or \code{NULL} (default): if integer, use only that many datapoints from the original dataset in the \code{data} argument.}

\item{prior}{Logical: Calculate the prior distribution? Default is \code{FALSE} unless \code{data} argument is missing or \code{NULL}.}

\item{startupMCiterations}{Integer, default 3600: number of initial Monte Carlo iterations.}

\item{minMCiterations}{Integer, default 0: minimum number of Monte Carlo iterations to be doneby a chain.}

\item{maxMCiterations}{Integer, default \code{Inf}: Do at most this many Monte Carlo iterations per chain.}

\item{maxhours}{Numeric, default \code{Inf}: approximate time limit, in hours, for the Monte Carlo computation to last.}

\item{ncheckpoints}{Integer or \code{NULL}, default 12: number of datapoints (per chain) to use for checking when the Monte Carlo computation should end. If \code{NULL}, this is equal to number of variates + 2. If Inf, use all datapoints.}

\item{maxrelMCSE}{Numeric positive, default \code{+Inf}: desired maximal \emph{relative Monte Carlo Standard Error} of calculated probabilities with respect to their variability with new data. The default \code{+Inf} means that \code{minESS} is used instead. \code{maxrelMCSE} is related to \code{minESS} by \eqn{\mathrm{maxrelMCSE} = 1/\sqrt{\mathrm{minESS} + \mathrm{initES}}}.}

\item{minESS}{Numeric positive or \code{NULL}, default 450: desired minimal Monte Carlo \emph{Expected Sample Size}. If \code{NULL}, it is equal to the final \code{nsamplesperchain}. \code{minESS} is related to \code{maxrelMCSE} by \eqn{\mathrm{minESS} = 1/\mathrm{maxrelMCSE}^2 - \mathrm{initES}}.}

\item{initES}{Numeric positive, default 2: number of initial "burn-in" samples, separated by the Expected Sample Size, to be discarded. Note that the Monte Carlo chain typically starts in a high-probability region, so there is no reason to discard many initial samples.}

\item{thinning}{Integer or \code{NULL} (default): thin out the Monte Carlo samples by this value. If \code{NULL}: let the diagnostics decide the thinning value.}

\item{verbose}{Logical, default \code{TRUE}: output the progress to terminal? If \code{FALSE}, the progress is outputted to the file \code{'main.log'} in the \code{outputdir} directory.}

\item{plottraces}{Logical, default \code{TRUE}: save plots of the Monte Carlo traces of diagnostic values?}

\item{showKtraces}{Logical, default \code{FALSE}: save plots of the Monte Carlo traces of the K parameter?}

\item{showAlphatraces}{Logical, default \code{FALSE}: save plots of the Monte Carlo traces of the Alpha parameter?}

\item{hyperparams}{List: hyperparameters of the hyperprior; see values in "Usage".}
}
\value{
A "learnt" object, or name of directory containing such an object and other output files, or \code{NULL}, depending on argument \code{valueislearnt}.

\code{learn()} saves several files in a directory. By default this output directory is a temporary directory within the one used by \code{\link[base:tempdir]{base::tempdir()}}, but an alternative one can be chosen with the argument \verb{outputdir =}. The output directory contain several diagnostic files for the Monte Carlo computation; in particular:
\itemize{
\item \code{MCtraces.pdf}: shows several trace plots of the Monte Carlo sampling; the correspondin data are in the file \code{MCtraces.rds}.
\item \code{plotsamples_learnt.pdf}, \code{plotquantiles_learnt.pdf}: show the marginal posterior distributions of each individual variate, together with their variability (as samples or quantiles).
\item \verb{log-1.log}, \verb{log-2.log}, ... one for each parallel core; report the progress of each parallel Monte Carlo computation and notes about it.
\item \code{rng_seed.rds}: the state of the pseudorandom seed (see \link[base:Random]{base::Random}) when \code{learn()} was called.
\item \code{metadata.csv}: a copy of the metadata.
}

It is recommended that you give an explicit argument \verb{outputdir =} and save the directory with the files above for future reference. In particular, the \code{MCtraces.pdf} plot and \code{MCtraces.rds} data can be useful to report Monte Carlo convergence in any work of yours that used \strong{Prova}.
}
\description{
Compute the posterior joint probability distribution of the variates conditional on the given data, by means of Markov-chain Monte Carlo, using the package \strong{Nimble}.
}
\details{
This function takes as main inputs a set of data and metadata, and computes the full joint probability distribution for new data, including its variability. From this full joint distribution any other distributions of interest can subsequently be computed; see \code{\link[=Pr]{Pr()}} and related functions. This computation can also be interpreted as an estimation of the full joint frequency distribution of the variates in the \emph{whole population}, beyond the sample data, together with its uncertainty. The computation allows for the use of datapoints with partially missing variables: imputation is automatically made. This imputation is \emph{principled}, made according to the rules of probability theory.

The output is a "learnt" object, typically saved in a \code{learnt.rds} file, which is used in all subsequent probabilistic computations. Other information about the computation is provided in logs and plots, saved in a directory specified by the user.

See \code{vignette('intro')} for introductory examples.

The computation is "non-parametric": probability or frequency distributions are not assumed to be Gaussian or of any other specific shape; no "model" is assumed. The mathematical representation of the space of joint frequency distributions follows ideas of Dunson & Bhattacharya (2011); see \href{https://github.com/pglpm/prova/raw/main/development/manual/pglpm2024-bayes_nonparam.pdf}{technical manual} for details.

The computation is done via Markov-chain Monte Carlo, using the package \href{https://cran.r-project.org/package=nimble}{\strong{Nimble}}. "Convergence" of the Monte Carlo computation is automatically assessed with methods described in Vehtari & al. (2021) and Kwon & al. (2025); see \href{https://github.com/pglpm/prova/raw/main/development/manual/pglpm2024-bayes_nonparam.pdf}{technical manual} for details. The default values for convergence require that all of the following three conditions be fulilled:
\itemize{
\item The computation's numerical error (Monte-Carlo Standard Error) for the posterior probability must be smaller than 4.7\% of the standard deviation of the posterior's variability.
\item The computation's numerical error for the 0.055- and 0.945-percentiles of the posterior's variability should be smaller than 4.7\% of the distance between them.
}

Typically this requirement leads to final results obtained with the \code{\link[=Pr]{Pr()}} function having at least two significant digits.

The \code{learn()} function can take hours or even days to perform its computations, depending on the size of the dataset, number of variates, and the (initially unknown) "shape" of the underlying probability distribution. For this reason it is typically called within an R script, executed via \link[utils:Rscript]{utils::Rscript}. For example, a script 'myscript.R' could have the following structure:

\if{html}{\out{<div class="sourceCode">}}\preformatted{library('prova')

learn(
  data = 'filename_with_data.csv', # CSV file containing the dataset
  metadata = 'filename_with_metadata.csv', # CSV file containing the metadata
  outputdir = 'some_directory', # path to output directory
  parallel = 8 # machine has more than 8 cores, so we use 8
  ## possibly other arguments to learn()
)
}\if{html}{\out{</div>}}

and then be called on a bash terminal with

\if{html}{\out{<div class="sourceCode">}}\preformatted{$ Rscript myscript.R > learnoutput.log 2>&1 &
}\if{html}{\out{</div>}}

with such a call, the file 'learnoutput.log' will contain information about how the computation is proceeding and the estimated end time.
}
\examples{
\donttest{
### WARNING: the following example, if run, might even take a minute or more.

## Create dataset with 3 points of variate 'V' for demonstration:
dataset <- data.frame(V = rnorm(n = 3))

## Create metadata file:
metadata <- data.frame(name = 'V', type = 'continuous')

## Learn from the data:
learnt <- learn(
  data = dataset, metadata = metadata,
      ## the following parameters are unrealistic
      ## only used to reduce computation time for this example
  nsamples = 10, nchains = 1,
  startupMCiterations = 10, maxMCiterations = 10,
  minESS = 0, initES = 0
)

## Check structure of `learnt` object:
str(learnt)
}

}
\references{
For the mathematical representation of the frequency space:
\itemize{
\item Dunson, Bhattacharya (2011): \emph{Nonparametric Bayes regression and classification through mixtures of product kernels} \url{doi:10.1093/acprof:oso/9780199694587.003.0005}.
\item Ishwaran, Zarepour (2002): \emph{Exact and approximate sum representations for the Dirichlet process} \url{doi:10.2307/3315951}.
\item Porta Mana \url{https://github.com/pglpm/prova/raw/main/development/manual/pglpm2024-bayes_nonparam.pdf}.
}

About Bayesian inference under exchangeability ("population inference"):
\itemize{
\item Lindley, Novick (1981): \emph{The role of exchangeability in inference}, \url{doi:10.1214/aos/1176345331}.
\item Bernardo, Smith (2000): \emph{Bayesian Theory}. Wiley \url{doi:10.1002/9780470316870}.
\item Porta Mana \url{https://github.com/pglpm/prova/raw/main/development/manual/pglpm2024-bayes_nonparam.pdf}.
}

About nonparametrics:
\itemize{
\item Müller et al. (2015): \emph{Nonparametric Bayesian inference}. IMS \url{doi:10.1007/978-3-319-18968-0}.
\item Hjort et al. (2010): \emph{Bayesian Nonparametrics}. Cambridge University Press \url{doi:10.1017/CBO9780511802478}.
}

About Markov-chain Monte Carlo and "convergence":
\itemize{
\item de Valpine, Paciorek, Turek, & al. (2026): \emph{NIMBLE: MCMC, Particle Filtering, and Programmable Hierarchical Modeling} \url{doi:10.5281/zenodo.1211190}, \url{https://cran.r-project.org/package=nimble}.
\item Kwon & al. (2025): \emph{MCMC stopping rules in latent variable modelling} \url{doi:10.1111/bmsp.12357}.
\item Vehtari & al. (2021): \emph{Rank-normalization, folding, and localization: an improved R-hat for assessing convergence of MCMC} \url{doi:10.1214/20-BA1221}.
\item Roy (2020): \emph{Convergence diagnostics for Markov chain Monte Carlo} \url{doi:10.1146/annurev-statistics-031219-041300}.
\item Gilks & al. (1998): \emph{Markov Chain Monte Carlo in Practice}. Chapman & Hall/CRC \url{doi:10.1201/b14835}.
\item D. J. C. MacKay (2005): \emph{Information Theory, Inference, and Learning Algorithms}. Cambridge University Press \url{https://www.inference.org.uk/itila/book.html}.
\item Porta Mana \url{https://github.com/pglpm/prova/raw/main/development/manual/pglpm2024-bayes_nonparam.pdf}.
}
}
\seealso{
\code{\link[=metadatatemplate]{metadatatemplate()}} to help writing metadata files.

\code{\link[=Pr]{Pr()}} to calculate probabilities, and \code{\link[=qPr]{qPr()}} to calculate quantiles, given the data processed by \code{learn()}.

\code{\link[=rPr]{rPr()}} to generate datapoints similar to the data processed by \code{learn()}.

\code{\link[=mutualinfo]{mutualinfo()}} to calculate mutual information given the data processed by \code{learn()}.

\code{\link[=pread.csv]{pread.csv()}} and \code{\link[=pwrite.csv]{pwrite.csv()}} to read and write CSV files in the format used by \code{learn()}.
}
