# Welcome to systemml’s documentation!

Contents:

## systemml package

### Subpackages

#### systemml.mllearn package

##### Submodules

##### systemml.mllearn.estimators module

class `systemml.mllearn.estimators.``LinearRegression`(sqlCtx, fit_intercept=True, max_iter=100, tol=1e-06, C=1.0, solver='newton-cg', transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLRegressor`

  Performs linear regression to model the relationship between one numerical response variable and one or more explanatory (feature) variables.

      >>> import numpy as np
      >>> from sklearn import datasets
      >>> from systemml.mllearn import LinearRegression
      >>> from pyspark.sql import SQLContext
      >>> # Load the diabetes dataset
      >>> diabetes = datasets.load_diabetes()
      >>> # Use only one feature
      >>> diabetes_X = diabetes.data[:, np.newaxis, 2]
      >>> # Split the data into training/testing sets
      >>> diabetes_X_train = diabetes_X[:-20]
      >>> diabetes_X_test = diabetes_X[-20:]
      >>> # Split the targets into training/testing sets
      >>> diabetes_y_train = diabetes.target[:-20]
      >>> diabetes_y_test = diabetes.target[-20:]
      >>> # Create linear regression object
      >>> regr = LinearRegression(sqlCtx, solver='newton-cg')
      >>> # Train the model using the training sets
      >>> regr.fit(diabetes_X_train, diabetes_y_train)
      >>> # The mean square error
      >>> print("Residual sum of squares: %.2f" % np.mean((regr.predict(diabetes_X_test) - diabetes_y_test) ** 2))

class `systemml.mllearn.estimators.``LogisticRegression`(sqlCtx, penalty='l2', fit_intercept=True, max_iter=100, max_inner_iter=0, tol=1e-06, C=1.0, solver='newton-cg', transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLClassifier`

  Performs both binomial and multinomial logistic regression.

  Scikit-learn way

      >>> from sklearn import datasets, neighbors
      >>> from systemml.mllearn import LogisticRegression
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> digits = datasets.load_digits()
      >>> X_digits = digits.data
      >>> y_digits = digits.target + 1
      >>> n_samples = len(X_digits)
      >>> X_train = X_digits[:.9 * n_samples]
      >>> y_train = y_digits[:.9 * n_samples]
      >>> X_test = X_digits[.9 * n_samples:]
      >>> y_test = y_digits[.9 * n_samples:]
      >>> logistic = LogisticRegression(sqlCtx)
      >>> print('LogisticRegression score: %f' % logistic.fit(X_train, y_train).score(X_test, y_test))

  MLPipeline way

      >>> from pyspark.ml import Pipeline
      >>> from systemml.mllearn import LogisticRegression
      >>> from pyspark.ml.feature import HashingTF, Tokenizer
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> training = sqlCtx.createDataFrame([
      >>>     (0L, "a b c d e spark", 1.0),
      >>>     (1L, "b d", 2.0),
      >>>     (2L, "spark f g h", 1.0),
      >>>     (3L, "hadoop mapreduce", 2.0),
      >>>     (4L, "b spark who", 1.0),
      >>>     (5L, "g d a y", 2.0),
      >>>     (6L, "spark fly", 1.0),
      >>>     (7L, "was mapreduce", 2.0),
      >>>     (8L, "e spark program", 1.0),
      >>>     (9L, "a e c l", 2.0),
      >>>     (10L, "spark compile", 1.0),
      >>>     (11L, "hadoop software", 2.0)
      >>> ], ["id", "text", "label"])
      >>> tokenizer = Tokenizer(inputCol="text", outputCol="words")
      >>> hashingTF = HashingTF(inputCol="words", outputCol="features", numFeatures=20)
      >>> lr = LogisticRegression(sqlCtx)
      >>> pipeline = Pipeline(stages=[tokenizer, hashingTF, lr])
      >>> model = pipeline.fit(training)
      >>> test = sqlCtx.createDataFrame([
      >>>     (12L, "spark i j k"),
      >>>     (13L, "l m n"),
      >>>     (14L, "mapreduce spark"),
      >>>     (15L, "apache hadoop")], ["id", "text"])
      >>> prediction = model.transform(test)
      >>> prediction.show()

class `systemml.mllearn.estimators.``SVM`(sqlCtx, fit_intercept=True, max_iter=100, tol=1e-06, C=1.0, is_multi_class=False, transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLClassifier`

  Performs both binary-class and multiclass SVM (Support Vector Machines).

      >>> from sklearn import datasets, neighbors
      >>> from systemml.mllearn import SVM
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> digits = datasets.load_digits()
      >>> X_digits = digits.data
      >>> y_digits = digits.target
      >>> n_samples = len(X_digits)
      >>> X_train = X_digits[:.9 * n_samples]
      >>> y_train = y_digits[:.9 * n_samples]
      >>> X_test = X_digits[.9 * n_samples:]
      >>> y_test = y_digits[.9 * n_samples:]
      >>> svm = SVM(sqlCtx, is_multi_class=True)
      >>> print('LogisticRegression score: %f' % svm.fit(X_train, y_train).score(X_test, y_test))

class `systemml.mllearn.estimators.``NaiveBayes`(sqlCtx, laplace=1.0, transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLClassifier`

  Performs Naive Bayes.

      >>> from sklearn.datasets import fetch_20newsgroups
      >>> from sklearn.feature_extraction.text import TfidfVectorizer
      >>> from systemml.mllearn import NaiveBayes
      >>> from sklearn import metrics
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']
      >>> newsgroups_train = fetch_20newsgroups(subset='train', categories=categories)
      >>> newsgroups_test = fetch_20newsgroups(subset='test', categories=categories)
      >>> vectorizer = TfidfVectorizer()
      >>> # Both vectors and vectors_test are SciPy CSR matrix
      >>> vectors = vectorizer.fit_transform(newsgroups_train.data)
      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> nb = NaiveBayes(sqlCtx)
      >>> nb.fit(vectors, newsgroups_train.target)
      >>> pred = nb.predict(vectors_test)
      >>> metrics.f1_score(newsgroups_test.target, pred, average='weighted')

##### Module contents

###### SystemML Algorithms

Classification Algorithms
LogisticRegression | Performs binomial and multinomial logistic regression
SVM | Performs both binary-class and multi-class SVM
NaiveBayes | Multinomial naive bayes classifier

Regression Algorithms
LinearRegression | Performs linear regression

class `systemml.mllearn.``LinearRegression`(sqlCtx, fit_intercept=True, max_iter=100, tol=1e-06, C=1.0, solver='newton-cg', transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLRegressor`

  Performs linear regression to model the relationship between one numerical response variable and one or more explanatory (feature) variables.

      >>> import numpy as np
      >>> from sklearn import datasets
      >>> from systemml.mllearn import LinearRegression
      >>> from pyspark.sql import SQLContext
      >>> # Load the diabetes dataset
      >>> diabetes = datasets.load_diabetes()
      >>> # Use only one feature
      >>> diabetes_X = diabetes.data[:, np.newaxis, 2]
      >>> # Split the data into training/testing sets
      >>> diabetes_X_train = diabetes_X[:-20]
      >>> diabetes_X_test = diabetes_X[-20:]
      >>> # Split the targets into training/testing sets
      >>> diabetes_y_train = diabetes.target[:-20]
      >>> diabetes_y_test = diabetes.target[-20:]
      >>> # Create linear regression object
      >>> regr = LinearRegression(sqlCtx, solver='newton-cg')
      >>> # Train the model using the training sets
      >>> regr.fit(diabetes_X_train, diabetes_y_train)
      >>> # The mean square error
      >>> print("Residual sum of squares: %.2f" % np.mean((regr.predict(diabetes_X_test) - diabetes_y_test) ** 2))

class `systemml.mllearn.``LogisticRegression`(sqlCtx, penalty='l2', fit_intercept=True, max_iter=100, max_inner_iter=0, tol=1e-06, C=1.0, solver='newton-cg', transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLClassifier`

  Performs both binomial and multinomial logistic regression.

  Scikit-learn way

      >>> from sklearn import datasets, neighbors
      >>> from systemml.mllearn import LogisticRegression
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> digits = datasets.load_digits()
      >>> X_digits = digits.data
      >>> y_digits = digits.target + 1
      >>> n_samples = len(X_digits)
      >>> X_train = X_digits[:.9 * n_samples]
      >>> y_train = y_digits[:.9 * n_samples]
      >>> X_test = X_digits[.9 * n_samples:]
      >>> y_test = y_digits[.9 * n_samples:]
      >>> logistic = LogisticRegression(sqlCtx)
      >>> print('LogisticRegression score: %f' % logistic.fit(X_train, y_train).score(X_test, y_test))

  MLPipeline way

      >>> from pyspark.ml import Pipeline
      >>> from systemml.mllearn import LogisticRegression
      >>> from pyspark.ml.feature import HashingTF, Tokenizer
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> training = sqlCtx.createDataFrame([
      >>>     (0L, "a b c d e spark", 1.0),
      >>>     (1L, "b d", 2.0),
      >>>     (2L, "spark f g h", 1.0),
      >>>     (3L, "hadoop mapreduce", 2.0),
      >>>     (4L, "b spark who", 1.0),
      >>>     (5L, "g d a y", 2.0),
      >>>     (6L, "spark fly", 1.0),
      >>>     (7L, "was mapreduce", 2.0),
      >>>     (8L, "e spark program", 1.0),
      >>>     (9L, "a e c l", 2.0),
      >>>     (10L, "spark compile", 1.0),
      >>>     (11L, "hadoop software", 2.0)
      >>> ], ["id", "text", "label"])
      >>> tokenizer = Tokenizer(inputCol="text", outputCol="words")
      >>> hashingTF = HashingTF(inputCol="words", outputCol="features", numFeatures=20)
      >>> lr = LogisticRegression(sqlCtx)
      >>> pipeline = Pipeline(stages=[tokenizer, hashingTF, lr])
      >>> model = pipeline.fit(training)
      >>> test = sqlCtx.createDataFrame([
      >>>     (12L, "spark i j k"),
      >>>     (13L, "l m n"),
      >>>     (14L, "mapreduce spark"),
      >>>     (15L, "apache hadoop")], ["id", "text"])
      >>> prediction = model.transform(test)
      >>> prediction.show()

class `systemml.mllearn.``SVM`(sqlCtx, fit_intercept=True, max_iter=100, tol=1e-06, C=1.0, is_multi_class=False, transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLClassifier`

  Performs both binary-class and multiclass SVM (Support Vector Machines).

      >>> from sklearn import datasets, neighbors
      >>> from systemml.mllearn import SVM
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> digits = datasets.load_digits()
      >>> X_digits = digits.data
      >>> y_digits = digits.target
      >>> n_samples = len(X_digits)
      >>> X_train = X_digits[:.9 * n_samples]
      >>> y_train = y_digits[:.9 * n_samples]
      >>> X_test = X_digits[.9 * n_samples:]
      >>> y_test = y_digits[.9 * n_samples:]
      >>> svm = SVM(sqlCtx, is_multi_class=True)
      >>> print('LogisticRegression score: %f' % svm.fit(X_train, y_train).score(X_test, y_test))

class `systemml.mllearn.``NaiveBayes`(sqlCtx, laplace=1.0, transferUsingDF=False)

  Bases: `systemml.mllearn.estimators.BaseSystemMLClassifier`

  Performs Naive Bayes.

      >>> from sklearn.datasets import fetch_20newsgroups
      >>> from sklearn.feature_extraction.text import TfidfVectorizer
      >>> from systemml.mllearn import NaiveBayes
      >>> from sklearn import metrics
      >>> from pyspark.sql import SQLContext
      >>> sqlCtx = SQLContext(sc)
      >>> categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']
      >>> newsgroups_train = fetch_20newsgroups(subset='train', categories=categories)
      >>> newsgroups_test = fetch_20newsgroups(subset='test', categories=categories)
      >>> vectorizer = TfidfVectorizer()
      >>> # Both vectors and vectors_test are SciPy CSR matrix
      >>> vectors = vectorizer.fit_transform(newsgroups_train.data)
      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> nb = NaiveBayes(sqlCtx)
      >>> nb.fit(vectors, newsgroups_train.target)
      >>> pred = nb.predict(vectors_test)
      >>> metrics.f1_score(newsgroups_test.target, pred, average='weighted')

#### systemml.random package

##### Submodules

##### systemml.random.sampling module

`systemml.random.sampling.``normal`(loc=0.0, scale=1.0, size=(1, 1), sparsity=1.0)

  Draw random samples from a normal (Gaussian) distribution.

  loc: Mean (“centre”) of the distribution. scale: Standard deviation (spread or “width”) of the distribution. size: Output shape (only tuple of length 2, i.e. (m, n), supported). sparsity: Sparsity (between 0.0 and 1.0).

      >>> import systemml as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)
      >>> from systemml import random
      >>> m1 = sml.random.normal(loc=3, scale=2, size=(3,3))
      >>> m1.toNumPy()
      array([[ 3.48857226,  6.17261819,  2.51167259],
             [ 3.60506708, -1.90266305,  3.97601633],
             [ 3.62245706,  5.9430881 ,  2.53070413]])

`systemml.random.sampling.``uniform`(low=0.0, high=1.0, size=(1, 1), sparsity=1.0)

  Draw samples from a uniform distribution.

  low: Lower boundary of the output interval. high: Upper boundary of the output interval. size: Output shape (only tuple of length 2, i.e. (m, n), supported). sparsity: Sparsity (between 0.0 and 1.0).

      >>> import systemml as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)
      >>> from systemml import random
      >>> m1 = sml.random.uniform(size=(3,3))
      >>> m1.toNumPy()
      array([[ 0.54511396,  0.11937437,  0.72975775],
             [ 0.14135946,  0.01944448,  0.52544478],
             [ 0.67582422,  0.87068849,  0.02766852]])

`systemml.random.sampling.``poisson`(lam=1.0, size=(1, 1), sparsity=1.0)

  Draw samples from a Poisson distribution.

  lam: Expectation of interval, should be > 0. size: Output shape (only tuple of length 2, i.e. (m, n), supported). sparsity: Sparsity (between 0.0 and 1.0).

      >>> import systemml as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)
      >>> from systemml import random
      >>> m1 = sml.random.poisson(lam=1, size=(3,3))
      >>> m1.toNumPy()
      array([[ 1.,  0.,  2.],
             [ 1.,  0.,  0.],
             [ 0.,  0.,  0.]])

##### Module contents

###### Random Number Generation

Univariate distributions
normal | Normal / Gaussian distribution.
poisson | Poisson distribution.
uniform | Uniform distribution.

`systemml.random.``normal`(loc=0.0, scale=1.0, size=(1, 1), sparsity=1.0)

  Draw random samples from a normal (Gaussian) distribution.

  loc: Mean (“centre”) of the distribution. scale: Standard deviation (spread or “width”) of the distribution. size: Output shape (only tuple of length 2, i.e. (m, n), supported). sparsity: Sparsity (between 0.0 and 1.0).

      >>> import systemml as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)
      >>> from systemml import random
      >>> m1 = sml.random.normal(loc=3, scale=2, size=(3,3))
      >>> m1.toNumPy()
      array([[ 3.48857226,  6.17261819,  2.51167259],
             [ 3.60506708, -1.90266305,  3.97601633],
             [ 3.62245706,  5.9430881 ,  2.53070413]])

`systemml.random.``uniform`(low=0.0, high=1.0, size=(1, 1), sparsity=1.0)

  Draw samples from a uniform distribution.

  low: Lower boundary of the output interval. high: Upper boundary of the output interval. size: Output shape (only tuple of length 2, i.e. (m, n), supported). sparsity: Sparsity (between 0.0 and 1.0).

      >>> import systemml as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)
      >>> from systemml import random
      >>> m1 = sml.random.uniform(size=(3,3))
      >>> m1.toNumPy()
      array([[ 0.54511396,  0.11937437,  0.72975775],
             [ 0.14135946,  0.01944448,  0.52544478],
             [ 0.67582422,  0.87068849,  0.02766852]])

`systemml.random.``poisson`(lam=1.0, size=(1, 1), sparsity=1.0)

  Draw samples from a Poisson distribution.

  lam: Expectation of interval, should be > 0. size: Output shape (only tuple of length 2, i.e. (m, n), supported). sparsity: Sparsity (between 0.0 and 1.0).

      >>> import systemml as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)
      >>> from systemml import random
      >>> m1 = sml.random.poisson(lam=1, size=(3,3))
      >>> m1.toNumPy()
      array([[ 1.,  0.,  2.],
             [ 1.,  0.,  0.],
             [ 0.,  0.,  0.]])

### Submodules

### systemml.converters module

`systemml.converters.``getNumCols`(numPyArr)

`systemml.converters.``convertToMatrixBlock`(sc, src)

`systemml.converters.``convertToNumPyArr`(sc, mb)

`systemml.converters.``convertToPandasDF`(X)

`systemml.converters.``convertToLabeledDF`(sqlCtx, X, y=None)

### systemml.defmatrix module

`systemml.defmatrix.``setSparkContext`(sc)

  Before using the matrix, the user needs to invoke this function.

  sc: SparkContext
    SparkContext

class `systemml.defmatrix.``matrix`(data, op=None)

  Bases: `object`

  matrix class is a python wrapper that implements basic matrix operators, matrix functions as well as converters to common Python types (for example: Numpy arrays, PySpark DataFrame and Pandas DataFrame).

  The operators supported are:

  1. Arithmetic operators: +, -, , /, //, %, * as well as dot (i.e. matrix multiplication)
  2. Indexing in the matrix
  3. Relational/Boolean operators: <, <=, >, >=, ==, !=, &, |

  In addition, following functions are supported for matrix:

  1. transpose
  2. Aggregation functions: sum, mean, var, sd, max, min, argmin, argmax, cumsum
  3. Global statistical built-In functions: exp, log, abs, sqrt, round, floor, ceil, sin, cos, tan, asin, acos, atan, sign, solve

  Note: an evaluated matrix contains a data field computed by eval method as DataFrame or NumPy array.

      >>> import SystemML as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)

  Welcome to Apache SystemML!

      >>> m1 = sml.matrix(np.ones((3,3)) + 2)
      >>> m2 = sml.matrix(np.ones((3,3)) + 3)
      >>> m2 = m1 * (m2 + m1)
      >>> m4 = 1.0 - m2
      >>> m4
      # This matrix (mVar5) is backed by below given PyDML script (which is not yet evaluated). To fetch the data of this matrix, invoke toNumPy() or toDF() or toPandas() methods.
      mVar1 = load(" ", format="csv")
      mVar2 = load(" ", format="csv")
      mVar3 = mVar2 + mVar1
      mVar4 = mVar1 * mVar3
      mVar5 = 1.0 - mVar4
      save(mVar5, " ")
      >>> m2.eval()
      >>> m2
      # This matrix (mVar4) is backed by NumPy array. To fetch the NumPy array, invoke toNumPy() method.
      >>> m4
      # This matrix (mVar5) is backed by below given PyDML script (which is not yet evaluated). To fetch the data of this matrix, invoke toNumPy() or toDF() or toPandas() methods.
      mVar4 = load(" ", format="csv")
      mVar5 = 1.0 - mVar4
      save(mVar5, " ")
      >>> m4.sum(axis=1).toNumPy()
      array([[-60.],
             [-60.],
             [-60.]])

  Design Decisions:

  1. Until eval() method is invoked, we create an AST (not exposed to the user) that consist of unevaluated operations and data required by those operations. As an anology, a spark user can treat eval() method similar to calling RDD.persist() followed by RDD.count().
  2. The AST consist of two kinds of nodes: either of type matrix or of type DMLOp. Both these classes expose _visit method, that helps in traversing the AST in DFS manner.
  3. A matrix object can either be evaluated or not. If evaluated, the attribute ‘data’ is set to one of the supported types (for example: NumPy array or DataFrame). In this case, the attribute ‘op’ is set to None. If not evaluated, the attribute ‘op’ which refers to one of the intermediate node of AST and if of type DMLOp. In this case, the attribute ‘data’ is set to None.

  5. DMLOp has an attribute ‘inputs’ which contains list of matrix objects or DMLOp.

  6. To simplify the traversal, every matrix object is considered immutable and an matrix operations creates a new matrix object. As an example: m1 = sml.matrix(np.ones((3,3))) creates a matrix object backed by ‘data=(np.ones((3,3))’. m1 = m1 * 2 will create a new matrix object which is now backed by ‘op=DMLOp( ... )’ whose input is earlier created matrix object.

  7. Left indexing (implemented in __setitem__ method) is a special case, where Python expects the existing object to be mutated. To ensure the above property, we make deep copy of existing object and point any references to the left-indexed matrix to the newly created object. Then the left-indexed matrix is set to be backed by DMLOp consisting of following pydml: left-indexed-matrix = new-deep-copied-matrix left-indexed-matrix[index] = value

  8. Please use m.printAST() and/or type m for debugging. Here is a sample session:

         >>> npm = np.ones((3,3))
         >>> m1 = sml.matrix(npm + 3)
         >>> m2 = sml.matrix(npm + 5)
         >>> m3 = m1 + m2
         >>> m3
         mVar2 = load(" ", format="csv")
         mVar1 = load(" ", format="csv")
         mVar3 = mVar1 + mVar2
         save(mVar3, " ")
         >>> m3.printAST()
         - [mVar3] (op).
           - [mVar1] (data).
           - [mVar2] (data).

  `argmax`(axis=None)

    Returns the indices of the maximum values along an axis.

    axis : int, optional (only axis=1, i.e. rowIndexMax is supported in this version)

  `argmin`(axis=None)

    Returns the indices of the minimum values along an axis.

    axis : int, optional (only axis=1, i.e. rowIndexMax is supported in this version)

  `cumsum`(axis=None)

    Returns the indices of the maximum values along an axis.

    axis : int, optional (only axis=0, i.e. cumsum along the rows is supported in this version)

  `dml` = []
  `dot`(other)

    Numpy way of performing matrix multiplication

  `eval`(outputDF=False)

    This is a convenience function that calls the global eval method

  `max`(axis=None)

    Compute the maximum value along the specified axis

    axis : int, optional

  `mean`(axis=None)

    Compute the arithmetic mean along the specified axis

    axis : int, optional

  `min`(axis=None)

    Compute the minimum value along the specified axis

    axis : int, optional

  `ml` = None
  `printAST`(numSpaces=0)

    Please use m.printAST() and/or type m for debugging. Here is a sample session:

        >>> npm = np.ones((3,3))
        >>> m1 = sml.matrix(npm + 3)
        >>> m2 = sml.matrix(npm + 5)
        >>> m3 = m1 + m2
        >>> m3
        mVar2 = load(" ", format="csv")
        mVar1 = load(" ", format="csv")
        mVar3 = mVar1 + mVar2
        save(mVar3, " ")
        >>> m3.printAST()
        - [mVar3] (op).
          - [mVar1] (data).
          - [mVar2] (data).

  `script` = None
  `sd`(axis=None)

    Compute the standard deviation along the specified axis

    axis : int, optional

  `sum`(axis=None)

    Compute the sum along the specified axis

    axis : int, optional

  `systemmlVarID` = 0
  `toDF`()

    This is a convenience function that calls the global eval method and then converts the matrix object into DataFrame.

  `toNumPy`()

    This is a convenience function that calls the global eval method and then converts the matrix object into NumPy array.

  `toPandas`()

    This is a convenience function that calls the global eval method and then converts the matrix object into Pandas DataFrame.

  `trace`()

    Return the sum of the cells of the main diagonal square matrix

  `transpose`()

    Transposes the matrix.

  `var`(axis=None)

    Compute the variance along the specified axis

    axis : int, optional

  `visited` = []

`systemml.defmatrix.``eval`(outputs, outputDF=False, execute=True)

  Executes the unevaluated DML script and computes the matrices specified by outputs.

  outputs: list of matrices or a matrix object outputDF: back the data of matrix as PySpark DataFrame

`systemml.defmatrix.``solve`(A, b)

  Computes the least squares solution for system of linear equations A %*% x = b

      >>> import numpy as np
      >>> from sklearn import datasets
      >>> import SystemML as sml
      >>> from pyspark.sql import SQLContext
      >>> diabetes = datasets.load_diabetes()
      >>> diabetes_X = diabetes.data[:, np.newaxis, 2]
      >>> X_train = diabetes_X[:-20]
      >>> X_test = diabetes_X[-20:]
      >>> y_train = diabetes.target[:-20]
      >>> y_test = diabetes.target[-20:]
      >>> sml.setSparkContext(sc)
      >>> X = sml.matrix(X_train)
      >>> y = sml.matrix(y_train)
      >>> A = X.transpose().dot(X)
      >>> b = X.transpose().dot(y)
      >>> beta = sml.solve(A, b).toNumPy()
      >>> y_predicted = X_test.dot(beta)
      >>> print('Residual sum of squares: %.2f' % np.mean((y_predicted - y_test) ** 2))
      Residual sum of squares: 25282.12

class `systemml.defmatrix.``DMLOp`(inputs, dml=None)

  Bases: `object`

  Represents an intermediate node of Abstract syntax tree created to generate the PyDML script

  `printAST`(numSpaces)

`systemml.defmatrix.``exp`(X)

`systemml.defmatrix.``log`(X, y=None)

`systemml.defmatrix.``abs`(X)

`systemml.defmatrix.``sqrt`(X)

`systemml.defmatrix.``round`(X)

`systemml.defmatrix.``floor`(X)

`systemml.defmatrix.``ceil`(X)

`systemml.defmatrix.``sin`(X)

`systemml.defmatrix.``cos`(X)

`systemml.defmatrix.``tan`(X)

`systemml.defmatrix.``asin`(X)

`systemml.defmatrix.``acos`(X)

`systemml.defmatrix.``atan`(X)

`systemml.defmatrix.``sign`(X)

### systemml.mlcontext module

class `systemml.mlcontext.``MLResults`(results, sc)

  Bases: `object`

  Wrapper around a Java ML Results object.

  results: JavaObject
    A Java MLResults object as returned by calling ml.execute().
  sc: SparkContext
    SparkContext
  `get`(*outputs)
    outputs: string, list of strings
      Output variables as defined inside the DML script.

class `systemml.mlcontext.``MLContext`(sc)

  Bases: `object`

  Wrapper around the new SystemML MLContext.

  sc: SparkContext
    SparkContext
  `execute`(script)

    Execute a DML / PyDML script.

    script: Script instance
      Script instance defined with the appropriate input and output variables.
    ml_results: MLResults
      MLResults instance.
  `setExplain`(explain)

    Explanation about the program. Mainly intended for developers.

    explain: boolean

  `setExplainLevel`(explainLevel)

    Set explain level.

    explainLevel: string
      Can be one of “hops”, “runtime”, “recompile_hops”, “recompile_runtime” or in the above in upper case.
  `setStatistics`(statistics)

    Whether or not to output statistics (such as execution time, elapsed time) about script executions.

    statistics: boolean

class `systemml.mlcontext.``Script`(scriptString, scriptType='dml')

  Bases: `object`

  Instance of a DML/PyDML Script.

  scriptString: string
    Can be either a file path to a DML script or a DML script itself.
  scriptType: string
    Script language, either “dml” for DML (R-like) or “pydml” for PyDML (Python-like).
  `input`(*args, **kwargs)
    args: name, value tuple
      where name is a string, and currently supported value formats are double, string, dataframe, rdd, and list of such object.
    kwargs: dict of name, value pairs
      To know what formats are supported for name and value, look above.
  `output`(*names)
    names: string, list of strings
      Output variables as defined inside the DML script.

`systemml.mlcontext.``dml`(scriptString)

  Create a dml script object based on a string.

  scriptString: string
    Can be a path to a dml script or a dml script itself.
  script: Script instance
    Instance of a script object.

`systemml.mlcontext.``pydml`(scriptString)

  Create a pydml script object based on a string.

  scriptString: string
    Can be a path to a pydml script or a pydml script itself.
  script: Script instance
    Instance of a script object.

`systemml.mlcontext.``getNumCols`(numPyArr)

`systemml.mlcontext.``convertToMatrixBlock`(sc, src)

`systemml.mlcontext.``convertToNumPyArr`(sc, mb)

`systemml.mlcontext.``convertToPandasDF`(X)

`systemml.mlcontext.``convertToLabeledDF`(sqlCtx, X, y=None)

### Module contents

class `systemml.``MLResults`(results, sc)

  Bases: `object`

  Wrapper around a Java ML Results object.

  results: JavaObject
    A Java MLResults object as returned by calling ml.execute().
  sc: SparkContext
    SparkContext
  `get`(*outputs)
    outputs: string, list of strings
      Output variables as defined inside the DML script.

class `systemml.``MLContext`(sc)

  Bases: `object`

  Wrapper around the new SystemML MLContext.

  sc: SparkContext
    SparkContext
  `execute`(script)

    Execute a DML / PyDML script.

    script: Script instance
      Script instance defined with the appropriate input and output variables.
    ml_results: MLResults
      MLResults instance.
  `setExplain`(explain)

    Explanation about the program. Mainly intended for developers.

    explain: boolean

  `setExplainLevel`(explainLevel)

    Set explain level.

    explainLevel: string
      Can be one of “hops”, “runtime”, “recompile_hops”, “recompile_runtime” or in the above in upper case.
  `setStatistics`(statistics)

    Whether or not to output statistics (such as execution time, elapsed time) about script executions.

    statistics: boolean

class `systemml.``Script`(scriptString, scriptType='dml')

  Bases: `object`

  Instance of a DML/PyDML Script.

  scriptString: string
    Can be either a file path to a DML script or a DML script itself.
  scriptType: string
    Script language, either “dml” for DML (R-like) or “pydml” for PyDML (Python-like).
  `input`(*args, **kwargs)
    args: name, value tuple
      where name is a string, and currently supported value formats are double, string, dataframe, rdd, and list of such object.
    kwargs: dict of name, value pairs
      To know what formats are supported for name and value, look above.
  `output`(*names)
    names: string, list of strings
      Output variables as defined inside the DML script.

`systemml.``dml`(scriptString)

  Create a dml script object based on a string.

  scriptString: string
    Can be a path to a dml script or a dml script itself.
  script: Script instance
    Instance of a script object.

`systemml.``pydml`(scriptString)

  Create a pydml script object based on a string.

  scriptString: string
    Can be a path to a pydml script or a pydml script itself.
  script: Script instance
    Instance of a script object.

`systemml.``setSparkContext`(sc)

  Before using the matrix, the user needs to invoke this function.

  sc: SparkContext
    SparkContext

class `systemml.``matrix`(data, op=None)

  Bases: `object`

  matrix class is a python wrapper that implements basic matrix operators, matrix functions as well as converters to common Python types (for example: Numpy arrays, PySpark DataFrame and Pandas DataFrame).

  The operators supported are:

  1. Arithmetic operators: +, -, , /, //, %, * as well as dot (i.e. matrix multiplication)
  2. Indexing in the matrix
  3. Relational/Boolean operators: <, <=, >, >=, ==, !=, &, |

  In addition, following functions are supported for matrix:

  1. transpose
  2. Aggregation functions: sum, mean, var, sd, max, min, argmin, argmax, cumsum
  3. Global statistical built-In functions: exp, log, abs, sqrt, round, floor, ceil, sin, cos, tan, asin, acos, atan, sign, solve

  Note: an evaluated matrix contains a data field computed by eval method as DataFrame or NumPy array.

      >>> import SystemML as sml
      >>> import numpy as np
      >>> sml.setSparkContext(sc)

  Welcome to Apache SystemML!

      >>> m1 = sml.matrix(np.ones((3,3)) + 2)
      >>> m2 = sml.matrix(np.ones((3,3)) + 3)
      >>> m2 = m1 * (m2 + m1)
      >>> m4 = 1.0 - m2
      >>> m4
      # This matrix (mVar5) is backed by below given PyDML script (which is not yet evaluated). To fetch the data of this matrix, invoke toNumPy() or toDF() or toPandas() methods.
      mVar1 = load(" ", format="csv")
      mVar2 = load(" ", format="csv")
      mVar3 = mVar2 + mVar1
      mVar4 = mVar1 * mVar3
      mVar5 = 1.0 - mVar4
      save(mVar5, " ")
      >>> m2.eval()
      >>> m2
      # This matrix (mVar4) is backed by NumPy array. To fetch the NumPy array, invoke toNumPy() method.
      >>> m4
      # This matrix (mVar5) is backed by below given PyDML script (which is not yet evaluated). To fetch the data of this matrix, invoke toNumPy() or toDF() or toPandas() methods.
      mVar4 = load(" ", format="csv")
      mVar5 = 1.0 - mVar4
      save(mVar5, " ")
      >>> m4.sum(axis=1).toNumPy()
      array([[-60.],
             [-60.],
             [-60.]])

  Design Decisions:

  1. Until eval() method is invoked, we create an AST (not exposed to the user) that consist of unevaluated operations and data required by those operations. As an anology, a spark user can treat eval() method similar to calling RDD.persist() followed by RDD.count().
  2. The AST consist of two kinds of nodes: either of type matrix or of type DMLOp. Both these classes expose _visit method, that helps in traversing the AST in DFS manner.
  3. A matrix object can either be evaluated or not. If evaluated, the attribute ‘data’ is set to one of the supported types (for example: NumPy array or DataFrame). In this case, the attribute ‘op’ is set to None. If not evaluated, the attribute ‘op’ which refers to one of the intermediate node of AST and if of type DMLOp. In this case, the attribute ‘data’ is set to None.

  5. DMLOp has an attribute ‘inputs’ which contains list of matrix objects or DMLOp.

  6. To simplify the traversal, every matrix object is considered immutable and an matrix operations creates a new matrix object. As an example: m1 = sml.matrix(np.ones((3,3))) creates a matrix object backed by ‘data=(np.ones((3,3))’. m1 = m1 * 2 will create a new matrix object which is now backed by ‘op=DMLOp( ... )’ whose input is earlier created matrix object.

  7. Left indexing (implemented in __setitem__ method) is a special case, where Python expects the existing object to be mutated. To ensure the above property, we make deep copy of existing object and point any references to the left-indexed matrix to the newly created object. Then the left-indexed matrix is set to be backed by DMLOp consisting of following pydml: left-indexed-matrix = new-deep-copied-matrix left-indexed-matrix[index] = value

  8. Please use m.printAST() and/or type m for debugging. Here is a sample session:

         >>> npm = np.ones((3,3))
         >>> m1 = sml.matrix(npm + 3)
         >>> m2 = sml.matrix(npm + 5)
         >>> m3 = m1 + m2
         >>> m3
         mVar2 = load(" ", format="csv")
         mVar1 = load(" ", format="csv")
         mVar3 = mVar1 + mVar2
         save(mVar3, " ")
         >>> m3.printAST()
         - [mVar3] (op).
           - [mVar1] (data).
           - [mVar2] (data).

  `argmax`(axis=None)

    Returns the indices of the maximum values along an axis.

    axis : int, optional (only axis=1, i.e. rowIndexMax is supported in this version)

  `argmin`(axis=None)

    Returns the indices of the minimum values along an axis.

    axis : int, optional (only axis=1, i.e. rowIndexMax is supported in this version)

  `cumsum`(axis=None)

    Returns the indices of the maximum values along an axis.

    axis : int, optional (only axis=0, i.e. cumsum along the rows is supported in this version)

  `dml` = []
  `dot`(other)

    Numpy way of performing matrix multiplication

  `eval`(outputDF=False)

    This is a convenience function that calls the global eval method

  `max`(axis=None)

    Compute the maximum value along the specified axis

    axis : int, optional

  `mean`(axis=None)

    Compute the arithmetic mean along the specified axis

    axis : int, optional

  `min`(axis=None)

    Compute the minimum value along the specified axis

    axis : int, optional

  `ml` = None
  `printAST`(numSpaces=0)

    Please use m.printAST() and/or type m for debugging. Here is a sample session:

        >>> npm = np.ones((3,3))
        >>> m1 = sml.matrix(npm + 3)
        >>> m2 = sml.matrix(npm + 5)
        >>> m3 = m1 + m2
        >>> m3
        mVar2 = load(" ", format="csv")
        mVar1 = load(" ", format="csv")
        mVar3 = mVar1 + mVar2
        save(mVar3, " ")
        >>> m3.printAST()
        - [mVar3] (op).
          - [mVar1] (data).
          - [mVar2] (data).

  `script` = None
  `sd`(axis=None)

    Compute the standard deviation along the specified axis

    axis : int, optional

  `sum`(axis=None)

    Compute the sum along the specified axis

    axis : int, optional

  `systemmlVarID` = 0
  `toDF`()

    This is a convenience function that calls the global eval method and then converts the matrix object into DataFrame.

  `toNumPy`()

    This is a convenience function that calls the global eval method and then converts the matrix object into NumPy array.

  `toPandas`()

    This is a convenience function that calls the global eval method and then converts the matrix object into Pandas DataFrame.

  `trace`()

    Return the sum of the cells of the main diagonal square matrix

  `transpose`()

    Transposes the matrix.

  `var`(axis=None)

    Compute the variance along the specified axis

    axis : int, optional

  `visited` = []

`systemml.``eval`(outputs, outputDF=False, execute=True)

  Executes the unevaluated DML script and computes the matrices specified by outputs.

  outputs: list of matrices or a matrix object outputDF: back the data of matrix as PySpark DataFrame

`systemml.``solve`(A, b)

  Computes the least squares solution for system of linear equations A %*% x = b

      >>> import numpy as np
      >>> from sklearn import datasets
      >>> import SystemML as sml
      >>> from pyspark.sql import SQLContext
      >>> diabetes = datasets.load_diabetes()
      >>> diabetes_X = diabetes.data[:, np.newaxis, 2]
      >>> X_train = diabetes_X[:-20]
      >>> X_test = diabetes_X[-20:]
      >>> y_train = diabetes.target[:-20]
      >>> y_test = diabetes.target[-20:]
      >>> sml.setSparkContext(sc)
      >>> X = sml.matrix(X_train)
      >>> y = sml.matrix(y_train)
      >>> A = X.transpose().dot(X)
      >>> b = X.transpose().dot(y)
      >>> beta = sml.solve(A, b).toNumPy()
      >>> y_predicted = X_test.dot(beta)
      >>> print('Residual sum of squares: %.2f' % np.mean((y_predicted - y_test) ** 2))
      Residual sum of squares: 25282.12

class `systemml.``DMLOp`(inputs, dml=None)

  Bases: `object`

  Represents an intermediate node of Abstract syntax tree created to generate the PyDML script

  `printAST`(numSpaces)

`systemml.``exp`(X)

`systemml.``log`(X, y=None)

`systemml.``abs`(X)

`systemml.``sqrt`(X)

`systemml.``round`(X)

`systemml.``floor`(X)

`systemml.``ceil`(X)

`systemml.``sin`(X)

`systemml.``cos`(X)

`systemml.``tan`(X)

`systemml.``asin`(X)

`systemml.``acos`(X)

`systemml.``atan`(X)

`systemml.``sign`(X)

`systemml.``getNumCols`(numPyArr)

`systemml.``convertToMatrixBlock`(sc, src)

`systemml.``convertToNumPyArr`(sc, mb)

`systemml.``convertToPandasDF`(X)

`systemml.``convertToLabeledDF`(sqlCtx, X, y=None)

# Indices and tables

- Index
- Module Index
- Search Page