Top

noxer.gm.wass_keras module

Implementation of Wasserstein discriminator as in https://arxiv.org/pdf/1701.07875.pdf and related publications.

This implementation is Keras based.

"""
Implementation of Wasserstein discriminator
as in https://arxiv.org/pdf/1701.07875.pdf and
related publications.

This implementation is Keras based.
"""

import numpy as np

from keras.layers import Input, Flatten, Dense, LeakyReLU, Reshape, \
    Activation, Convolution2D, Deconvolution2D, MaxPool2D, UpSampling2D, \
    BatchNormalization, Concatenate

from keras.models import Model
from keras.optimizers import Adam, RMSprop
from keras.losses import mean_squared_error, mean_absolute_error

from sklearn.base import BaseEstimator, RegressorMixin
from sklearn.model_selection import train_test_split

from keras.constraints import Constraint
from keras import backend as K


class WeightClip(Constraint):
    '''Clips the weights incident to each hidden unit to be
    inside a range.
    '''
    def __init__(self, c=0.01):
        self.c = c

    def __call__(self, p):
        return K.clip(p, -self.c, self.c)

    def get_config(self):
        return {'name': self.__class__.__name__,
                'c': self.c}


def wass_train_loss(y_true, y_pred):
    ''' maximizes score for positive true labels
     and minimizes for negative true'''
    return K.sum(-y_true * y_pred)


def _check_wass_weights(model):
    # sanity check on weights of the discriminator
    for layer in model.layers:
        weights = layer.get_weights()  # list of numpy arrays
        for w in weights:
            m = np.max(np.abs(w))
            print(m)


class WassersteinDiscriminator(BaseEstimator, RegressorMixin):
    """This class learns the objective function that
    discriminates between two distributions of samples.
    For a particular sample, such objective estimates
    how fake the sample looks. The less the value of
    objective - the more realistic the sample looks."""
    def __init__(self):
        self.model = None

    def make_architecture(self, X, Y):
        """
        Make the discriminator network.

        X: n-d array of samples
            Data to train on.
        Y: n-d array of binary values
            Label for every sample: +1 (real sample), -1 (fake sample)
        """

        sh = X[0].shape
        ip = Input(shape=sh)
        h = ip
        h = Flatten()(h) if len(sh) > 1 else h

        for i in range(2):
            h = Dense(128, W_constraint=WeightClip(), bias_constraint=WeightClip())(h)
            h = LeakyReLU()(h)

        # final output - single score value
        h = Dense(1, W_constraint=WeightClip(), bias_constraint=WeightClip())(h)

        self.model = Model(inputs=ip, outputs=h)
        return self

    def compile_architecture(self, X, Y):
        self.model.compile(
            optimizer=RMSprop(),
            loss=wass_train_loss
        )
        return self

    def train(self, X, Y, monitor=None):
        """
        Run the training on compiled model

        X: n-d array of samples
            Data to train on.

        Y: n-d array of binary values
            Label for every sample: +1 (real sample), -1 (fake sample)
        """
        for i in range(10):
            self.model.fit(X, Y, epochs=1)

            if monitor is not None:
                monitor()

    def fit(self, X, Y, monitor=None):
        """
        Fit the Wasserstein GAN discriminator to the data

        X: n-d array of samples
            Data to train on.

        Y: n-d array of binary values
            Label for every sample: +1 (real sample), -1 (fake sample)
        """
        self.make_architecture(X, Y)
        self.compile_architecture(X, Y)
        self.train(X, Y, monitor)

    def predict(self, X):
        """
        Makes estimations with Wasserstein GAN discriminator

        X: n-d array of samples
            Data to train on.

        Returns
        -------

        Y: n-d array of scores
            Score which indicates how far the sample
            is from being in "real" category
        """

        return self.model.predict(X)

Functions

def wass_train_loss(

y_true, y_pred)

maximizes score for positive true labels and minimizes for negative true

def wass_train_loss(y_true, y_pred):
    ''' maximizes score for positive true labels
     and minimizes for negative true'''
    return K.sum(-y_true * y_pred)

Classes

class WassersteinDiscriminator

This class learns the objective function that discriminates between two distributions of samples. For a particular sample, such objective estimates how fake the sample looks. The less the value of objective - the more realistic the sample looks.

class WassersteinDiscriminator(BaseEstimator, RegressorMixin):
    """This class learns the objective function that
    discriminates between two distributions of samples.
    For a particular sample, such objective estimates
    how fake the sample looks. The less the value of
    objective - the more realistic the sample looks."""
    def __init__(self):
        self.model = None

    def make_architecture(self, X, Y):
        """
        Make the discriminator network.

        X: n-d array of samples
            Data to train on.
        Y: n-d array of binary values
            Label for every sample: +1 (real sample), -1 (fake sample)
        """

        sh = X[0].shape
        ip = Input(shape=sh)
        h = ip
        h = Flatten()(h) if len(sh) > 1 else h

        for i in range(2):
            h = Dense(128, W_constraint=WeightClip(), bias_constraint=WeightClip())(h)
            h = LeakyReLU()(h)

        # final output - single score value
        h = Dense(1, W_constraint=WeightClip(), bias_constraint=WeightClip())(h)

        self.model = Model(inputs=ip, outputs=h)
        return self

    def compile_architecture(self, X, Y):
        self.model.compile(
            optimizer=RMSprop(),
            loss=wass_train_loss
        )
        return self

    def train(self, X, Y, monitor=None):
        """
        Run the training on compiled model

        X: n-d array of samples
            Data to train on.

        Y: n-d array of binary values
            Label for every sample: +1 (real sample), -1 (fake sample)
        """
        for i in range(10):
            self.model.fit(X, Y, epochs=1)

            if monitor is not None:
                monitor()

    def fit(self, X, Y, monitor=None):
        """
        Fit the Wasserstein GAN discriminator to the data

        X: n-d array of samples
            Data to train on.

        Y: n-d array of binary values
            Label for every sample: +1 (real sample), -1 (fake sample)
        """
        self.make_architecture(X, Y)
        self.compile_architecture(X, Y)
        self.train(X, Y, monitor)

    def predict(self, X):
        """
        Makes estimations with Wasserstein GAN discriminator

        X: n-d array of samples
            Data to train on.

        Returns
        -------

        Y: n-d array of scores
            Score which indicates how far the sample
            is from being in "real" category
        """

        return self.model.predict(X)

Ancestors (in MRO)

Static methods

def __init__(

self)

Initialize self. See help(type(self)) for accurate signature.

def __init__(self):
    self.model = None

def compile_architecture(

self, X, Y)

def compile_architecture(self, X, Y):
    self.model.compile(
        optimizer=RMSprop(),
        loss=wass_train_loss
    )
    return self

def fit(

self, X, Y, monitor=None)

Fit the Wasserstein GAN discriminator to the data

X: n-d array of samples Data to train on.

Y: n-d array of binary values Label for every sample: +1 (real sample), -1 (fake sample)

def fit(self, X, Y, monitor=None):
    """
    Fit the Wasserstein GAN discriminator to the data
    X: n-d array of samples
        Data to train on.
    Y: n-d array of binary values
        Label for every sample: +1 (real sample), -1 (fake sample)
    """
    self.make_architecture(X, Y)
    self.compile_architecture(X, Y)
    self.train(X, Y, monitor)

def get_params(

self, deep=True)

Get parameters for this estimator.

Parameters

deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators.

Returns

params : mapping of string to any Parameter names mapped to their values.

def get_params(self, deep=True):
    """Get parameters for this estimator.
    Parameters
    ----------
    deep : boolean, optional
        If True, will return the parameters for this estimator and
        contained subobjects that are estimators.
    Returns
    -------
    params : mapping of string to any
        Parameter names mapped to their values.
    """
    out = dict()
    for key in self._get_param_names():
        # We need deprecation warnings to always be on in order to
        # catch deprecated param values.
        # This is set in utils/__init__.py but it gets overwritten
        # when running under python3 somehow.
        warnings.simplefilter("always", DeprecationWarning)
        try:
            with warnings.catch_warnings(record=True) as w:
                value = getattr(self, key, None)
            if len(w) and w[0].category == DeprecationWarning:
                # if the parameter is deprecated, don't show it
                continue
        finally:
            warnings.filters.pop(0)
        # XXX: should we rather test if instance of estimator?
        if deep and hasattr(value, 'get_params'):
            deep_items = value.get_params().items()
            out.update((key + '__' + k, val) for k, val in deep_items)
        out[key] = value
    return out

def make_architecture(

self, X, Y)

Make the discriminator network.

X: n-d array of samples Data to train on. Y: n-d array of binary values Label for every sample: +1 (real sample), -1 (fake sample)

def make_architecture(self, X, Y):
    """
    Make the discriminator network.
    X: n-d array of samples
        Data to train on.
    Y: n-d array of binary values
        Label for every sample: +1 (real sample), -1 (fake sample)
    """
    sh = X[0].shape
    ip = Input(shape=sh)
    h = ip
    h = Flatten()(h) if len(sh) > 1 else h
    for i in range(2):
        h = Dense(128, W_constraint=WeightClip(), bias_constraint=WeightClip())(h)
        h = LeakyReLU()(h)
    # final output - single score value
    h = Dense(1, W_constraint=WeightClip(), bias_constraint=WeightClip())(h)
    self.model = Model(inputs=ip, outputs=h)
    return self

def predict(

self, X)

Makes estimations with Wasserstein GAN discriminator

X: n-d array of samples Data to train on.

Returns

Y: n-d array of scores Score which indicates how far the sample is from being in "real" category

def predict(self, X):
    """
    Makes estimations with Wasserstein GAN discriminator
    X: n-d array of samples
        Data to train on.
    Returns
    -------
    Y: n-d array of scores
        Score which indicates how far the sample
        is from being in "real" category
    """
    return self.model.predict(X)

def score(

self, X, y, sample_weight=None)

Returns the coefficient of determination R^2 of the prediction.

The coefficient R^2 is defined as (1 - u/v), where u is the residual sum of squares ((y_true - y_pred) 2).sum() and v is the total sum of squares ((y_true - y_true.mean()) 2).sum(). The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y, disregarding the input features, would get a R^2 score of 0.0.

Parameters

X : array-like, shape = (n_samples, n_features) Test samples.

y : array-like, shape = (n_samples) or (n_samples, n_outputs) True values for X.

sample_weight : array-like, shape = [n_samples], optional Sample weights.

Returns

score : float R^2 of self.predict(X) wrt. y.

def score(self, X, y, sample_weight=None):
    """Returns the coefficient of determination R^2 of the prediction.
    The coefficient R^2 is defined as (1 - u/v), where u is the residual
    sum of squares ((y_true - y_pred) ** 2).sum() and v is the total
    sum of squares ((y_true - y_true.mean()) ** 2).sum().
    The best possible score is 1.0 and it can be negative (because the
    model can be arbitrarily worse). A constant model that always
    predicts the expected value of y, disregarding the input features,
    would get a R^2 score of 0.0.
    Parameters
    ----------
    X : array-like, shape = (n_samples, n_features)
        Test samples.
    y : array-like, shape = (n_samples) or (n_samples, n_outputs)
        True values for X.
    sample_weight : array-like, shape = [n_samples], optional
        Sample weights.
    Returns
    -------
    score : float
        R^2 of self.predict(X) wrt. y.
    """
    from .metrics import r2_score
    return r2_score(y, self.predict(X), sample_weight=sample_weight,
                    multioutput='variance_weighted')

def set_params(

self, **params)

Set the parameters of this estimator.

The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object.

Returns

self

def set_params(self, **params):
    """Set the parameters of this estimator.
    The method works on simple estimators as well as on nested objects
    (such as pipelines). The latter have parameters of the form
    ``<component>__<parameter>`` so that it's possible to update each
    component of a nested object.
    Returns
    -------
    self
    """
    if not params:
        # Simple optimization to gain speed (inspect is slow)
        return self
    valid_params = self.get_params(deep=True)
    nested_params = defaultdict(dict)  # grouped by prefix
    for key, value in params.items():
        key, delim, sub_key = key.partition('__')
        if key not in valid_params:
            raise ValueError('Invalid parameter %s for estimator %s. '
                             'Check the list of available parameters '
                             'with `estimator.get_params().keys()`.' %
                             (key, self))
        if delim:
            nested_params[key][sub_key] = value
        else:
            setattr(self, key, value)
    for key, sub_params in nested_params.items():
        valid_params[key].set_params(**sub_params)
    return self

def train(

self, X, Y, monitor=None)

Run the training on compiled model

X: n-d array of samples Data to train on.

Y: n-d array of binary values Label for every sample: +1 (real sample), -1 (fake sample)

def train(self, X, Y, monitor=None):
    """
    Run the training on compiled model
    X: n-d array of samples
        Data to train on.
    Y: n-d array of binary values
        Label for every sample: +1 (real sample), -1 (fake sample)
    """
    for i in range(10):
        self.model.fit(X, Y, epochs=1)
        if monitor is not None:
            monitor()

Instance variables

var model

class WeightClip

Clips the weights incident to each hidden unit to be inside a range.

class WeightClip(Constraint):
    '''Clips the weights incident to each hidden unit to be
    inside a range.
    '''
    def __init__(self, c=0.01):
        self.c = c

    def __call__(self, p):
        return K.clip(p, -self.c, self.c)

    def get_config(self):
        return {'name': self.__class__.__name__,
                'c': self.c}

Ancestors (in MRO)

  • WeightClip
  • keras.constraints.Constraint
  • builtins.object

Static methods

def __init__(

self, c=0.01)

Initialize self. See help(type(self)) for accurate signature.

def __init__(self, c=0.01):
    self.c = c

def get_config(

self)

def get_config(self):
    return {'name': self.__class__.__name__,
            'c': self.c}

Instance variables

var c