[SPARK-18929][ML] Add Tweedie distribution in GLM #16344
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -48,7 +48,7 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam | |
| /** | ||
| * Param for the name of family which is a description of the error distribution | ||
| * to be used in the model. | ||
| * Supported options: "gaussian", "binomial", "poisson", "gamma" and "tweedie". | ||
| * Default is "gaussian". | ||
| * | ||
| * @group param | ||
|
|
@@ -63,6 +63,28 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam | |
| @Since("2.0.0") | ||
| def getFamily: String = $(family) | ||
|
|
||
| /** | ||
| * Param for the power in the variance function of the Tweedie distribution which provides | ||
| * the relationship between the variance and mean of the distribution. | ||
| * Used only for the Tweedie family. | ||
|
||
| * (see <a href="https://en.wikipedia.org/wiki/Tweedie_distribution"> | ||
| * Tweedie Distribution (Wikipedia)</a>) | ||
| * Supported value: 0 and [1, Inf). Note that when the value of the variance power is | ||
| * 0, 1, or 2, the Gaussian, Poisson or Gamma family is used, respectively. | ||
| * | ||
| * @group param | ||
| */ | ||
| @Since("2.2.0") | ||
| final val variancePower: Param[Double] = new Param(this, "variancePower", | ||
| "The power in the variance function of the Tweedie distribution which characterizes " + | ||
| "the relationship between the variance and mean of the distribution. " + | ||
| "Used for the Tweedie family. Supported value: 0 and [1, Inf).", | ||
|
Contributor
There was a problem hiding this comment.
|
||
| (x: Double) => x >= 1.0 || x == 0.0) | ||
|
|
||
| /** @group getParam */ | ||
| @Since("2.2.0") | ||
| def getVariancePower: Double = $(variancePower) | ||
|
|
||
| /** | ||
| * Param for the name of link function which provides the relationship | ||
| * between the linear predictor and the mean of the distribution function. | ||
|
|
@@ -108,8 +130,9 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam | |
| featuresDataType: DataType): StructType = { | ||
| if (isDefined(link)) { | ||
| require(supportedFamilyAndLinkPairs.contains( | ||
| $(family) -> Link.fromName($(link))), | ||
| s"Generalized Linear Regression with ${$(family)} family " + | ||
| s"does not support ${$(link)} link function.") | ||
| } | ||
| val newSchema = super.validateAndTransformSchema(schema, fitting, featuresDataType) | ||
| if (hasLinkPredictionCol) { | ||
|
|
@@ -128,13 +151,14 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam | |
| * Generalized linear model (Wikipedia)</a>) | ||
| * specified by giving a symbolic description of the linear | ||
| * predictor (link function) and a description of the error distribution (family). | ||
| * It supports "gaussian", "binomial", "poisson", "gamma" and "tweedie" as family. | ||
| * Valid link functions for each family is listed below. The first link function of each family | ||
| * is the default one. | ||
| * - "gaussian" : "identity", "log", "inverse" | ||
| * - "binomial" : "logit", "probit", "cloglog" | ||
| * - "poisson" : "log", "identity", "sqrt" | ||
| * - "gamma" : "inverse", "identity", "log" | ||
| * - "tweedie" : "identity", "log" | ||
|
||
| */ | ||
| @Experimental | ||
| @Since("2.0.0") | ||
|
|
@@ -157,6 +181,16 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val | |
| def setFamily(value: String): this.type = set(family, value) | ||
| setDefault(family -> Gaussian.name) | ||
|
|
||
| /** | ||
| * Sets the value of param [[variancePower]]. | ||
| * Used only when family is "tweedie". | ||
| * | ||
| * @group setParam | ||
| */ | ||
| @Since("2.2.0") | ||
| def setVariancePower(value: Double): this.type = set(variancePower, value) | ||
| setDefault(variancePower -> 1.5) | ||
|
Contributor
There was a problem hiding this comment. Why set the default value to 1.5, AFAIK, R set the default produces the same model with h2o.glm has the consistent default values with R, should we keep consistent with them?
Contributor
Author
There was a problem hiding this comment. Done. change default variancePower to 0.0, which will use Gaussian (with default identity link) |
||
|
|
||
| /** | ||
| * Sets the value of param [[link]]. | ||
| * | ||
|
|
@@ -242,7 +276,7 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val | |
| def setLinkPredictionCol(value: String): this.type = set(linkPredictionCol, value) | ||
|
|
||
| override protected def train(dataset: Dataset[_]): GeneralizedLinearRegressionModel = { | ||
| val familyObj = Family.fromModel(this) | ||
| val linkObj = if (isDefined(link)) { | ||
| Link.fromName($(link)) | ||
| } else { | ||
|
|
@@ -303,20 +337,24 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine | |
|
|
||
| /** Set of family and link pairs that GeneralizedLinearRegression supports. */ | ||
| private[regression] lazy val supportedFamilyAndLinkPairs = Set( | ||
| "gaussian" -> Identity, "gaussian" -> Log, "gaussian" -> Inverse, | ||
|
||
| "binomial" -> Logit, "binomial" -> Probit, "binomial" -> CLogLog, | ||
| "poisson" -> Log, "poisson" -> Identity, "poisson" -> Sqrt, | ||
| "gamma" -> Inverse, "gamma" -> Identity, "gamma" -> Log, | ||
| "tweedie" -> Identity, "tweedie" -> Log | ||
| ) | ||
|
|
||
| /** Set of family names that GeneralizedLinearRegression supports. */ | ||
| private[regression] lazy val supportedFamilyNames = supportedFamilyAndLinkPairs.map(_._1) | ||
|
|
||
| /** Set of link names that GeneralizedLinearRegression supports. */ | ||
| private[regression] lazy val supportedLinkNames = supportedFamilyAndLinkPairs.map(_._2.name) | ||
|
|
||
| private[regression] val epsilon: Double = 1E-16 | ||
|
|
||
| /** Constant used in initialization and deviance to avoid numerical issues. */ | ||
| private[regression] val delta: Double = 0.1 | ||
|
||
|
|
||
| /** | ||
| * Wrapper of family and link combination used in the model. | ||
| */ | ||
|
|
@@ -365,7 +403,6 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine | |
| /** | ||
| * A description of the error distribution to be used in the model. | ||
| * | ||
| */ | ||
| private[regression] abstract class Family(val name: String) extends Serializable { | ||
|
|
||
|
|
@@ -397,41 +434,113 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine | |
|
|
||
| /** Trim the fitted value so that it will be in valid range. */ | ||
| def project(mu: Double): Double = mu | ||
|
|
||
| } | ||
|
|
||
| private[regression] object Family { | ||
|
|
||
| /** | ||
| * Gets the [[Family]] object based on family and variancePower. | ||
| * 1) retrieve object based on family name | ||
| * 2) if family name is tweedie, retrieve object based on variancePower | ||
|
||
| * | ||
| * @param model a GenerealizedLinearRegressionBase object | ||
| */ | ||
| def fromModel(model: GeneralizedLinearRegressionBase): Family = { | ||
|
||
| model.getFamily match { | ||
| case "gaussian" => Gaussian | ||
|
||
| case "binomial" => Binomial | ||
| case "poisson" => Poisson | ||
| case "gamma" => Gamma | ||
| case "tweedie" => | ||
| model.getVariancePower match { | ||
| case 0.0 => Gaussian | ||
| case 1.0 => Poisson | ||
| case 2.0 => Gamma | ||
| case default => new TweedieFamily(default) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Tweedie exponential family distribution. | ||
| * This includes the special cases of Gaussian, Poisson and Gamma. | ||
| */ | ||
| private[regression] class TweedieFamily(private val variancePower: Double) | ||
|
||
| extends Family("tweedie") { | ||
|
|
||
| /* | ||
| The canonical link is 1 - variancePower. Except for the special cases of Gaussian, | ||
|
||
| Poisson and Gamma, the canonical link is rarely used. Set Log as the default link. | ||
| */ | ||
| override val defaultLink: Link = Log | ||
|
||
|
|
||
| override def initialize(y: Double, weight: Double): Double = { | ||
| if (variancePower >= 1.0 && variancePower < 2.0) { | ||
| require(y >= 0.0, s"The response variable of the specified distribution " + | ||
|
||
| s"should be non-negative, but got $y") | ||
| } else if (variancePower >= 2.0) { | ||
| require(y > 0.0, s"The response variable of the specified distribution " + | ||
|
||
| s"should be non-negative, but got $y") | ||
|
||
| } | ||
| if (y == 0) delta else y | ||
| } | ||
|
|
||
| override def variance(mu: Double): Double = math.pow(mu, variancePower) | ||
|
|
||
| private def yp(y: Double, mu: Double, p: Double): Double = { | ||
| if (p == 0) { | ||
| math.log(y / mu) | ||
| } else { | ||
| (math.pow(y, p) - math.pow(mu, p)) / p | ||
| } | ||
| } | ||
|
|
||
| override def deviance(y: Double, mu: Double, weight: Double): Double = { | ||
| // Force y >= delta for Poisson or compound Poisson | ||
| val y1 = if (variancePower >= 1.0 && variancePower < 2.0) { | ||
| math.max(y, delta) | ||
| } else { | ||
| y | ||
| } | ||
| 2.0 * weight * | ||
| (y * yp(y1, mu, 1.0 - variancePower) - yp(y, mu, 2.0 - variancePower)) | ||
| } | ||
|
|
||
| override def aic( | ||
| predictions: RDD[(Double, Double, Double)], | ||
| deviance: Double, | ||
| numInstances: Double, | ||
| weightSum: Double): Double = { | ||
| /* | ||
| This depends on the density of the Tweedie distribution. | ||
| Only implemented for Gaussian, Poisson and Gamma at this point. | ||
| */ | ||
| throw new UnsupportedOperationException("No AIC available for the tweedie family") | ||
| } | ||
|
|
||
| override def project(mu: Double): Double = { | ||
| if (mu < epsilon) { | ||
| epsilon | ||
| } else if (mu.isInfinity) { | ||
| Double.MaxValue | ||
| } else { | ||
| mu | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Gaussian exponential family distribution. | ||
| * The default link for the Gaussian family is the identity link. | ||
| */ | ||
| private[regression] object Gaussian extends TweedieFamily(0.0) { | ||
|
||
|
|
||
| override val name: String = "gaussian" | ||
|
|
||
| override val defaultLink: Link = Identity | ||
|
|
||
| override def aic( | ||
| predictions: RDD[(Double, Double, Double)], | ||
| deviance: Double, | ||
|
|
@@ -508,25 +617,11 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine | |
| * Poisson exponential family distribution. | ||
| * The default link for the Poisson family is the log link. | ||
| */ | ||
| private[regression] object Poisson extends TweedieFamily(1.0) { | ||
|
|
||
| override val name: String = "poisson" | ||
|
|
||
| override val defaultLink: Link = Log | ||
|
|
||
| override def aic( | ||
| predictions: RDD[(Double, Double, Double)], | ||
|
|
@@ -537,37 +632,17 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine | |
| weight * dist.Poisson(mu).logProbabilityOf(y.toInt) | ||
| }.sum() | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Gamma exponential family distribution. | ||
| * The default link for the Gamma family is the inverse link. | ||
| */ | ||
| private[regression] object Gamma extends TweedieFamily(2.0) { | ||
|
|
||
| override val name: String = "gamma" | ||
|
|
||
| override val defaultLink: Link = Inverse | ||
|
|
||
| override def aic( | ||
| predictions: RDD[(Double, Double, Double)], | ||
|
|
@@ -579,16 +654,6 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine | |
| weight * dist.Gamma(1.0 / disp, mu * disp).logPdf(y) | ||
| }.sum() + 2.0 | ||
| } | ||
| } | ||
|
|
||
| /** | ||
|
|
@@ -720,7 +785,8 @@ class GeneralizedLinearRegressionModel private[ml] ( | |
|
|
||
| import GeneralizedLinearRegression._ | ||
|
|
||
| private lazy val familyObj = Family.fromModel(this) | ||
|
|
||
| private lazy val linkObj = if (isDefined(link)) { | ||
| Link.fromName($(link)) | ||
| } else { | ||
|
|
@@ -905,7 +971,8 @@ class GeneralizedLinearRegressionSummary private[regression] ( | |
| */ | ||
| @Since("2.0.0") @transient val predictions: DataFrame = model.transform(dataset) | ||
|
|
||
| private[regression] lazy val family: Family = Family.fromModel(model) | ||
|
|
||
| private[regression] lazy val link: Link = if (model.isDefined(model.link)) { | ||
| Link.fromName(model.getLink) | ||
| } else { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nits: Param -> parameter, tweedie -> Tweedie (two lines below).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
changed tweedie. but other docs have been using Param..