Packages

t

cloudflow.spark

SparkStreamlet

trait SparkStreamlet extends Streamlet[SparkStreamletContext] with Serializable

The base class for defining Spark streamlets. Derived classes need to override createLogic to provide the custom implementation for the behavior of the streamlet.

Here's an example:

// new custom `SparkStreamlet`
object MySparkProcessor extends SparkStreamlet {
  // Step 1: Define inlets and outlets. Note for the outlet you need to specify
  //         the partitioner function explicitly
  val in = AvroInlet[Data]("in")
  val out = AvroOutlet[Simple]("out", _.name)

  // Step 2: Define the shape of the streamlet. In this example the streamlet
  //         has 1 inlet and 1 outlet
  val shape = StreamletShape(in, out)

  // Step 3: Provide custom implementation of `SparkStreamletLogic` that defines
  //         the behavior of the streamlet
  override def createLogic() = new SparkStreamletLogic {
    override def buildStreamingQueries = {
      val dataset = readStream(in)
      val outStream = dataset.select($"name").as[Simple]
      val query = writeStream(outStream, out, OutputMode.Append)
      Seq(query)
    }
  }
}
Linear Supertypes
Serializable, Serializable, Streamlet[SparkStreamletContext], AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SparkStreamlet
  2. Serializable
  3. Serializable
  4. Streamlet
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Abstract Value Members

  1. abstract def createLogic(): SparkStreamletLogic
    Attributes
    protected
  2. abstract def shape(): StreamletShape
    Definition Classes
    Streamlet

Concrete Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val StopTimeout: FiniteDuration
  5. val applicationName: String
  6. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  7. def attributes: Set[StreamletAttribute]
    Attributes
    protected[cloudflow]
    Definition Classes
    Streamlet
  8. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native() @HotSpotIntrinsicCandidate()
  9. def configParameters: IndexedSeq[ConfigParameter]
    Definition Classes
    Streamlet
  10. final def configuredValue(context: SparkStreamletContext, configKey: String): String
  11. implicit final def context: SparkStreamletContext
    Attributes
    protected
    Definition Classes
    Streamlet
  12. final def createContext(config: Config): SparkStreamletContext
    Attributes
    protected
    Definition Classes
    SparkStreamlet → Streamlet
  13. def customAttributes: Set[StreamletAttribute]
    Definition Classes
    Streamlet
  14. def defineConfigParameters(): Array[ConfigParameter]
    Definition Classes
    Streamlet
  15. def defineCustomAttributes(): Array[StreamletAttribute]
    Definition Classes
    Streamlet
  16. def defineVolumeMounts(): Array[VolumeMount]
    Definition Classes
    Streamlet
  17. def description: String
    Definition Classes
    Streamlet
  18. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  19. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  20. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  21. final def getContext(): SparkStreamletContext
    Attributes
    protected
    Definition Classes
    Streamlet
  22. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  23. final def inlets: IndexedSeq[Inlet]
    Definition Classes
    Streamlet
  24. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  25. final def jsonDescriptor: String
    Definition Classes
    Streamlet
  26. def labels: IndexedSeq[String]
    Definition Classes
    Streamlet
  27. lazy val log: Logger
    Attributes
    protected
    Definition Classes
    Streamlet
    Annotations
    @transient()
  28. def logStartRunnerMessage(buildInfo: String): Unit
    Definition Classes
    SparkStreamlet → Streamlet
  29. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  30. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  31. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  32. final def outlets: IndexedSeq[Outlet]
    Definition Classes
    Streamlet
  33. final def run(context: SparkStreamletContext): StreamletExecution
    Definition Classes
    SparkStreamlet → Streamlet
  34. final val runtime: SparkStreamletRuntime.type
    Definition Classes
    SparkStreamlet → Streamlet
  35. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  36. def toString(): String
    Definition Classes
    AnyRef → Any
  37. def updateSparkSession(session: SparkSession): SparkSession

    Override this method to modify the org.apache.spark.SparkSession used in this SparkStreamlet.

    Override this method to modify the org.apache.spark.SparkSession used in this SparkStreamlet. By default this method does not modify the session.

  38. def volumeMounts: IndexedSeq[VolumeMount]
    Definition Classes
    Streamlet
  39. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  40. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  41. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated
    Deprecated

Inherited from Serializable

Inherited from Serializable

Inherited from Streamlet[SparkStreamletContext]

Inherited from AnyRef

Inherited from Any

Ungrouped